<commit_before>import config
import re
import os
import logging

log = logging.getLogger('urlCache')

FILENAME_MAX_LENGTH = 100 # maxlength of filenames
class FileCache(object):
    def __init__(self, dir, subdirs = [], log = None):
        ''' subdirs must be an array '''
        for i in xrange(0, len(subdirs)):
            dir = os.path.join(dir, self.create_filename(subdirs[i]))
        self.path = dir
        # create the path only if we write something there, thats why those variables getting set
        if os.path.isdir(self.path) is False:
            self.create_path = True
        else:
            self.create_path = False

    @staticmethod
    def create_filename(s):
        return re.sub('[^a-zA-Z0-9]','_',s)

    def get_path(self, section, create = False):
        if self.create_path:
            if create:
                try:
                    os.makedirs(self.path)
                except:
                    pass
            else:
                return None
        self.create_path = False
        return os.path.join(self.path, section)

    def remove(self, section):
        raise Exception("TODO implement")

    def lookup(self, section):
        file = self.get_path(section)
        if file and os.path.isfile(file):
            log.debug('using cache [%s] path: %s' % (section, file))
            f = open(file, 'r')
            return ''.join(f.readlines())
        return None

    def lookup_size(self, section):
        file = self.get_path(section)
        if file and os.path.isfile(file):
            return os.path.getsize(file)
        return None

    def read_stream(self, section):
        file = self.get_path(section)
        if file:
            return open(file, 'rb')
        return None

    def truncate(self, section, x):
        file = self.get_path(section)
        if file:
            a = open(file, 'r+b')
            a.truncate(x)

    def get_stream(self, section):
        file = self.get_path(section, True)
        return open(file, 'wb')

    def get_append_stream(self, section):
        file = self.get_path(section, True)
        return open(file, 'ab')

    def write(self, section, data):
        file = self.get_path(section, True)
        open(file, 'w').writelines(data)

Cache = FileCache

try:
    from kyotocabinet import *
except:
    config.cachePort = 0
    pass
else:
    dbList = {}
    class KyotoCache(object):
        def __init__(self, dir, subdirs = [], log = None):
            if dir not in dbList:
                dbList[dir] = DB()
                dbList[dir].open(dir+".kch#z=lzma", DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC)
            self.db = dbList[dir]
            self.key = "/".join(subdirs)

        def lookup(self, section):
            ret = self.db.get(self.key+"/"+section)
            return ret
        def write(self, section, data):
            self.db.set(self.key+"/"+section, data)
        def remove(self, section):
            self.db.remove(self.key+"/"+section)

        def lookup_size(self, section):
            raise Exception
        def read_stream(self, section):
            raise Exception
        def truncate(self, section, x):
            raise Exception
        def get_stream(self, section):
            raise Exception
        def get_append_stream(self, section):
            raise Exception

    Cache = KyotoCache

if config.cachePort:
    from socket import *
    import pickle
    import time

    HOST = 'localhost'
    PORT = config.cachePort
    ADDR = (HOST,PORT)

    # we have a cacheserver - write a client for it
    class CacheClient(object):
        def __init__(self, dir, subdirs = [], log = None):
            self.dir = dir
            self.key = "/".join(subdirs)

        def lookup(self, section):
            return self.sendRecv('lookup', section)
        def remove(self, section):
            return self.sendRecv('remove', section)
        def write(self, section, data):
            return self.sendRecv('write', section, data)

        def sendRecv(self, command, section, value=''):
            s = socket(AF_INET,SOCK_STREAM)
            for i in range(0,60):
                #s.connect((ADDR))
                try:
                    s.connect((ADDR))
                except:
                    time.sleep(1)
                    log.warning("couldn't connect to cache server")
                else:
                    break
            data = pickle.dumps({'c':command,'k':self.key,'section':section,'d':self.dir,'v':value})
            size = str(len(data))
            size += (8-len(size))*" "
            try:
                s.send(size+data)
            except:
                print data
            retdata = ''
            if command == 'lookup':
                try:
                    size = int(s.recv(8).rstrip())
                except:
                    return None
                if size:
                    retdata = ''
                    while size > 0:
                        try:
                            chunk = s.recv(size)
                        except:
                            return None
                        if chunk == '':
                            break
                        retdata += chunk
                        size -= len(chunk)
                    try:
                        retdata = pickle.loads(retdata)
                    except:
                        print retdata
                else:
                    retdata = None
            s.close()
            return retdata

    Cache = CacheClient
<commit_msg>connect just once to cache server<commit_after>import config
import re
import os
import logging

log = logging.getLogger('urlCache')

FILENAME_MAX_LENGTH = 100 # maxlength of filenames
class FileCache(object):
    def __init__(self, dir, subdirs = [], log = None):
        ''' subdirs must be an array '''
        for i in xrange(0, len(subdirs)):
            dir = os.path.join(dir, self.create_filename(subdirs[i]))
        self.path = dir
        # create the path only if we write something there, thats why those variables getting set
        if os.path.isdir(self.path) is False:
            self.create_path = True
        else:
            self.create_path = False

    @staticmethod
    def create_filename(s):
        return re.sub('[^a-zA-Z0-9]','_',s)

    def get_path(self, section, create = False):
        if self.create_path:
            if create:
                try:
                    os.makedirs(self.path)
                except:
                    pass
            else:
                return None
        self.create_path = False
        return os.path.join(self.path, section)

    def remove(self, section):
        raise Exception("TODO implement")

    def lookup(self, section):
        file = self.get_path(section)
        if file and os.path.isfile(file):
            log.debug('using cache [%s] path: %s' % (section, file))
            f = open(file, 'r')
            return ''.join(f.readlines())
        return None

    def lookup_size(self, section):
        file = self.get_path(section)
        if file and os.path.isfile(file):
            return os.path.getsize(file)
        return None

    def read_stream(self, section):
        file = self.get_path(section)
        if file:
            return open(file, 'rb')
        return None

    def truncate(self, section, x):
        file = self.get_path(section)
        if file:
            a = open(file, 'r+b')
            a.truncate(x)

    def get_stream(self, section):
        file = self.get_path(section, True)
        return open(file, 'wb')

    def get_append_stream(self, section):
        file = self.get_path(section, True)
        return open(file, 'ab')

    def write(self, section, data):
        file = self.get_path(section, True)
        open(file, 'w').writelines(data)

Cache = FileCache

try:
    from kyotocabinet import *
except:
    config.cachePort = 0
    pass
else:
    dbList = {}
    class KyotoCache(object):
        def __init__(self, dir, subdirs = [], log = None):
            if dir not in dbList:
                dbList[dir] = DB()
                dbList[dir].open(dir+".kch#z=lzma", DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC)
            self.db = dbList[dir]
            self.key = "/".join(subdirs)

        def lookup(self, section):
            ret = self.db.get(self.key+"/"+section)
            return ret
        def write(self, section, data):
            self.db.set(self.key+"/"+section, data)
        def remove(self, section):
            self.db.remove(self.key+"/"+section)

        def lookup_size(self, section):
            raise Exception
        def read_stream(self, section):
            raise Exception
        def truncate(self, section, x):
            raise Exception
        def get_stream(self, section):
            raise Exception
        def get_append_stream(self, section):
            raise Exception

    Cache = KyotoCache

if config.cachePort:
    from socket import *
    import pickle
    import time

    HOST = 'localhost'
    PORT = config.cachePort
    ADDR = (HOST,PORT)

    conList = {}

    # we have a cacheserver - write a client for it
    class CacheClient(object):
        def __init__(self, dir, subdirs = []):
            self.dir = dir
            self.setKey(subdirs)
            if dir not in conList:
                s = socket(AF_INET,SOCK_STREAM)
                conList[dir] = s
                try:
                    s.connect((ADDR))
                except:
                    time.sleep(1)
                    log.warning("couldn't connect to cache server")
            self.c = conList[dir]

        def setKey(self, subdirs = []):
            self.key = "/".join(subdirs)

        def lookup(self, section):
            return self.sendRecv('lookup', section)
        def remove(self, section):
            return self.sendRecv('remove', section)
        def write(self, section, data):
            return self.sendRecv('write', section, data)

        def sendRecv(self, command, section, value=''):
            data = pickle.dumps({'c':command,'k':self.key,'section':section,'d':self.dir,'v':value})
            size = str(len(data))
            size += (8-len(size))*" "
            try:
                self.c.send(size+data)
            except:
                print data
            retdata = ''
            if command == 'lookup':
                try:
                    size = int(self.c.recv(8).rstrip())
                except:
                    log.error("err")
                    return None
                if size:
                    retdata = ''
                    while size > 0:
                        try:
                            chunk = self.c.recv(size)
                        except:
                            log.error("err")
                            return None
                        if chunk == '':
                            break
                        retdata += chunk
                        size -= len(chunk)
                    try:
                        retdata = pickle.loads(retdata)
                    except:
                        log.error("err")
                        print retdata
                else:
                    retdata = None
            return retdata

    Cache = CacheClient
<|endoftext|>


<commit_before>"""
Utils for CMake
Author: https://github.com/klivelinux
"""

import os
import re
import utils
import rtconfig

def GenerateCFiles(env,project):
    """
    Generate CMakeLists.txt files
    """

    info = utils.ProjectInfo(env)

    CC = os.path.join(rtconfig.EXEC_PATH, rtconfig.CC)
    CXX = os.path.join(rtconfig.EXEC_PATH, rtconfig.CXX)
    AS = os.path.join(rtconfig.EXEC_PATH, rtconfig.AS)
    AR = os.path.join(rtconfig.EXEC_PATH, rtconfig.AR)
    LINK = os.path.join(rtconfig.EXEC_PATH, rtconfig.LINK)
    SIZE = os.path.join(rtconfig.EXEC_PATH, rtconfig.SIZE)
    OBJDUMP = os.path.join(rtconfig.EXEC_PATH, rtconfig.OBJDUMP)
    OBJCOPY = os.path.join(rtconfig.EXEC_PATH, rtconfig.OBJCPY)

    cm_file = open('CMakeLists.txt', 'w')
    if cm_file:
        cm_file.write("CMAKE_MINIMUM_REQUIRED(VERSION 3.10)\n\n")

        cm_file.write("SET(CMAKE_SYSTEM_NAME Generic)\n")
        cm_file.write("#SET(CMAKE_VERBOSE_MAKEFILE ON)\n\n")

        cm_file.write("SET(CMAKE_C_COMPILER \""+ CC + "\")\n")
        cm_file.write("SET(CMAKE_CXX_COMPILER \""+ CXX + "\")\n")
        cm_file.write("SET(CMAKE_ASM_COMPILER \""+ AS + "\")\n")
        cm_file.write("SET(CMAKE_OBJCOPY \""+ OBJCOPY + "\")\n")
        cm_file.write("SET(CMAKE_SIZE \""+ SIZE + "\")\n\n")

        cm_file.write("SET(CMAKE_C_FLAGS \""+ rtconfig.CFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_CXX_FLAGS \""+ rtconfig.CXXFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_ASM_FLAGS \""+ rtconfig.AFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_EXE_LINKER_FLAGS \""+ re.sub('-T(\s*)', '-T ${CMAKE_SOURCE_DIR}/',rtconfig.LFLAGS) + "\")\n\n")

        cm_file.write("SET(CMAKE_CXX_STANDARD 14)\n")
        cm_file.write("PROJECT(rtthread C CXX ASM)\n")

        cm_file.write("INCLUDE_DIRECTORIES(\n")
        for i in info['CPPPATH']:
                cm_file.write( "\t" +i + "\n")
        cm_file.write(")\n\n")

        cm_file.write("ADD_DEFINITIONS(\n")
        for i in info['CPPDEFINES']:
                cm_file.write("\t-D" + i + "\n")
        cm_file.write(")\n\n")

        cm_file.write("SET(PROJECT_SOURCES\n")
        for group in project:
            for f in group['src']:
                cm_file.write( "\t"+os.path.normpath(f.rfile().abspath)+"\n" )
        cm_file.write(")\n\n")

        cm_file.write("LINK_DIRECTORIES(\n")
        for group in project:
            if 'LIBPATH' in group.keys():
                for f in group['LIBPATH']:
                    cm_file.write( "\t"+ f + "\n" )
        cm_file.write(")\n\n")

        cm_file.write("LINK_LIBRARIES(\n")
        for group in project:
            if 'LIBS' in group.keys():
                for f in group['LIBS']:
                    cm_file.write( "\t"+ "{}\n".format(f))
        cm_file.write(")\n\n")

        cm_file.write("ADD_EXECUTABLE(${CMAKE_PROJECT_NAME}.elf ${PROJECT_SOURCES})\n")
        cm_file.write("ADD_CUSTOM_COMMAND(TARGET ${CMAKE_PROJECT_NAME}.elf POST_BUILD \nCOMMAND ${CMAKE_OBJCOPY} -O binary ${CMAKE_PROJECT_NAME}.elf ${CMAKE_PROJECT_NAME}.bin COMMAND ${CMAKE_SIZE} ${CMAKE_PROJECT_NAME}.elf)")

        cm_file.close()

    return

def CMakeProject(env,project):
    print('Update setting files for CMakeLists.txt...')
    GenerateCFiles(env,project)
    print('Done!')

    return
<commit_msg>[update] support windows cmake tool.<commit_after>"""
Utils for CMake
Author: https://github.com/klivelinux
"""

import os
import sys
import re
import utils
import rtconfig

def GenerateCFiles(env,project):
    """
    Generate CMakeLists.txt files
    """
    info = utils.ProjectInfo(env)

    CC = os.path.join(rtconfig.EXEC_PATH, rtconfig.CC).replace('\\', "/")
    CXX = os.path.join(rtconfig.EXEC_PATH, rtconfig.CXX).replace('\\', "/")
    AS = os.path.join(rtconfig.EXEC_PATH, rtconfig.AS).replace('\\', "/")
    AR = os.path.join(rtconfig.EXEC_PATH, rtconfig.AR).replace('\\', "/")
    LINK = os.path.join(rtconfig.EXEC_PATH, rtconfig.LINK).replace('\\', "/")
    SIZE = os.path.join(rtconfig.EXEC_PATH, rtconfig.SIZE).replace('\\', "/")
    OBJDUMP = os.path.join(rtconfig.EXEC_PATH, rtconfig.OBJDUMP).replace('\\', "/")
    OBJCOPY = os.path.join(rtconfig.EXEC_PATH, rtconfig.OBJCPY).replace('\\', "/")

    if "win32" in sys.platform:
        CC += ".exe"
        CXX += ".exe"
        AS += ".exe"
        AR += ".exe"
        LINK += ".exe"
        SIZE += ".exe"
        OBJDUMP += ".exe"
        OBJCOPY += ".exe"

    cm_file = open('CMakeLists.txt', 'w')
    if cm_file:
        cm_file.write("CMAKE_MINIMUM_REQUIRED(VERSION 3.10)\n\n")

        cm_file.write("SET(CMAKE_SYSTEM_NAME Generic)\n")
        cm_file.write("#SET(CMAKE_VERBOSE_MAKEFILE ON)\n\n")

        cm_file.write("SET(CMAKE_C_COMPILER \""+ CC + "\")\n")
        cm_file.write("SET(CMAKE_CXX_COMPILER \""+ CXX + "\")\n")
        cm_file.write("SET(CMAKE_ASM_COMPILER \""+ AS + "\")\n")
        cm_file.write("SET(CMAKE_OBJCOPY \""+ OBJCOPY + "\")\n")
        cm_file.write("SET(CMAKE_SIZE \""+ SIZE + "\")\n\n")

        cm_file.write("SET(CMAKE_C_FLAGS \""+ rtconfig.CFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_CXX_FLAGS \""+ rtconfig.CXXFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_ASM_FLAGS \""+ rtconfig.AFLAGS + "\")\n")
        cm_file.write("SET(CMAKE_EXE_LINKER_FLAGS \""+ re.sub('-T(\s*)', '-T ${CMAKE_SOURCE_DIR}/',rtconfig.LFLAGS) + "\")\n\n")

        cm_file.write("SET(CMAKE_CXX_STANDARD 14)\n")
        cm_file.write("PROJECT(rtthread C CXX ASM)\n")

        cm_file.write("INCLUDE_DIRECTORIES(\n")
        for i in info['CPPPATH']:
                cm_file.write( "\t" + i.replace("\\", "/") + "\n")
        cm_file.write(")\n\n")

        cm_file.write("ADD_DEFINITIONS(\n")
        for i in info['CPPDEFINES']:
                cm_file.write("\t-D" + i + "\n")
        cm_file.write(")\n\n")

        cm_file.write("SET(PROJECT_SOURCES\n")
        for group in project:
            for f in group['src']:
                cm_file.write( "\t" + os.path.normpath(f.rfile().abspath).replace("\\", "/") + "\n" )
        cm_file.write(")\n\n")

        cm_file.write("LINK_DIRECTORIES(\n")
        for group in project:
            if 'LIBPATH' in group.keys():
                for f in group['LIBPATH']:
                    cm_file.write( "\t"+ f.replace("\\", "/") + "\n" )
        cm_file.write(")\n\n")

        cm_file.write("LINK_LIBRARIES(\n")
        for group in project:
            if 'LIBS' in group.keys():
                for f in group['LIBS']:
                    cm_file.write( "\t"+ "{}\n".format(f.replace("\\", "/")))
        cm_file.write(")\n\n")

        cm_file.write("ADD_EXECUTABLE(${CMAKE_PROJECT_NAME}.elf ${PROJECT_SOURCES})\n")
        cm_file.write("ADD_CUSTOM_COMMAND(TARGET ${CMAKE_PROJECT_NAME}.elf POST_BUILD \nCOMMAND ${CMAKE_OBJCOPY} -O binary ${CMAKE_PROJECT_NAME}.elf ${CMAKE_PROJECT_NAME}.bin COMMAND ${CMAKE_SIZE} ${CMAKE_PROJECT_NAME}.elf)")

        cm_file.close()

    return

def CMakeProject(env,project):
    print('Update setting files for CMakeLists.txt...')
    GenerateCFiles(env,project)
    print('Done!')

    return
<|endoftext|>


<commit_before># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains gsutil base integration test case class."""

from contextlib import contextmanager
import logging
import subprocess
import sys

import boto
from boto.exception import StorageResponseError
import gslib
from gslib.project_id import GOOG_PROJ_ID_HDR
from gslib.project_id import PopulateProjectId
from gslib.tests.testcase import base
import gslib.tests.util as util
from gslib.tests.util import RUN_S3_TESTS
from gslib.tests.util import SetBotoConfigForTest
from gslib.tests.util import unittest
from gslib.util import IS_WINDOWS
from gslib.util import Retry

LOGGER = logging.getLogger('integration-test')

# Contents of boto config file that will tell gsutil not to override the real
# error message with a warning about anonymous access if no credentials are
# provided in the config file.
BOTO_CONFIG_CONTENTS_IGNORE_ANON_WARNING = """
[Tests]
bypass_anonymous_access_warning = True
"""

def SkipForGS(reason):
  if not RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func

def SkipForS3(reason):
  if RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func

@unittest.skipUnless(util.RUN_INTEGRATION_TESTS,
                     'Not running integration tests.')
class GsUtilIntegrationTestCase(base.GsUtilTestCase):
  """Base class for gsutil integration tests."""
  GROUP_TEST_ADDRESS = 'gs-discussion@googlegroups.com'
  GROUP_TEST_ID = (
      '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5')
  USER_TEST_ADDRESS = 'gs-team@google.com'
  USER_TEST_ID = (
      '00b4903a9703325c6bfc98992d72e75600387a64b3b6bee9ef74613ef8842080')
  DOMAIN_TEST = 'google.com'
  # No one can create this bucket without owning the gmail.com domain, and we
  # won't create this bucket, so it shouldn't exist.
  # It would be nice to use google.com here but JSON API disallows
  # 'google' in resource IDs.
  nonexistent_bucket_name = 'nonexistent-bucket-foobar.gmail.com'

  def setUp(self):
    """Creates base configuration for integration tests."""
    super(GsUtilIntegrationTestCase, self).setUp()
    self.bucket_uris = []

    # Set up API version and project ID handler.
    self.api_version = boto.config.get_value(
        'GSUtil', 'default_api_version', '1')

    if util.RUN_S3_TESTS:
      self.test_api = 'XML'
      self.default_provider = 's3'
      self.nonexistent_bucket_name = (
          'nonexistentbucket-asf801rj3r9as90mfnnkjxpo02')
    else:
      # TODO: gsutil-beta: This should consider whether the command
      # supports the API or not, but since commands will default to XML
      # it is enough to assume XML support and override as needed in the tests.
      self.test_api = boto.config.get('GSUtil', 'force_api', 'JSON').upper()
      self.default_provider = 'gs'

  # Retry with an exponential backoff if a server error is received. This
  # ensures that we try *really* hard to clean up after ourselves.
  # TODO: As long as we're still using boto to do the teardown,
  # we decorate with boto exceptions.  Eventually this should be migrated
  # to CloudApi exceptions.
  @Retry(StorageResponseError, tries=6, timeout_secs=1)
  def tearDown(self):
    super(GsUtilIntegrationTestCase, self).tearDown()

    while self.bucket_uris:
      bucket_uri = self.bucket_uris[-1]
      try:
        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
      except StorageResponseError, e:
        # This can happen for tests of rm -r command, which for bucket-only
        # URIs delete the bucket at the end.
        if e.status == 404:
          self.bucket_uris.pop()
          continue
        else:
          raise
      while bucket_list:
        error = None
        for k in bucket_list:
          try:
            k.delete()
          except StorageResponseError, e:
            # This could happen if objects that have already been deleted are
            # still showing up in the listing due to eventual consistency. In
            # that case, we continue on until we've tried to deleted every
            # object in the listing before raising the error on which to retry.
            if e.status == 404:
              error = e
            else:
              raise
        if error:
          raise error  # pylint: disable=raising-bad-type
        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
      bucket_uri.delete_bucket()
      self.bucket_uris.pop()

  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
                   provider=None):
    """Creates a test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.
      storage_class: storage class to use. If not provided we us standard.
      provider: Provider to use - either "gs" (the default) or "s3".

    Returns:
      StorageUri for the created bucket.
    """
    if not provider:
      provider = self.default_provider
    bucket_name = bucket_name or self.MakeTempName('bucket')

    bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()),
                                  suppress_consec_slashes=False)

    if provider == 'gs':
      # Apply API version and project ID headers if necessary.
      headers = {'x-goog-api-version': self.api_version}
      headers[GOOG_PROJ_ID_HDR] = PopulateProjectId()
    else:
      headers = {}

    bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
    self.bucket_uris.append(bucket_uri)
    for i in range(test_objects):
      self.CreateObject(bucket_uri=bucket_uri,
                        object_name=self.MakeTempName('obj'),
                        contents='test %d' % i)
    return bucket_uri

  def CreateVersionedBucket(self, bucket_name=None, test_objects=0):
    """Creates a versioned test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.

    Returns:
      StorageUri for the created bucket with versioning enabled.
    """
    bucket_uri = self.CreateBucket(bucket_name=bucket_name,
                                   test_objects=test_objects)
    bucket_uri.configure_versioning(True)
    return bucket_uri

  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
    """Creates a test object.

    Args:
      bucket_uri: The URI of the bucket to place the object in. If not
                  specified, a new temporary bucket is created.
      object_name: The name to use for the object. If not specified, a temporary
                   test object name is constructed.
      contents: The contents to write to the object. If not specified, the key
                is not written to, which means that it isn't actually created
                yet on the server.

    Returns:
      A StorageUri for the created object.
    """
    bucket_uri = bucket_uri or self.CreateBucket()
    object_name = object_name or self.MakeTempName('obj')
    key_uri = bucket_uri.clone_replace_name(object_name)
    if contents is not None:
      key_uri.set_contents_from_string(contents)
    else:
      if self.default_provider == 's3':
        # S3 will only create the object if it has content.
        key_uri.set_contents_from_string('default')
    return key_uri

  def RunGsUtil(self, cmd, return_status=False, return_stdout=False,
                return_stderr=False, expected_status=0, stdin=None):
    """Runs the gsutil command.

    Args:
      cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
      return_status: If True, the exit status code is returned.
      return_stdout: If True, the standard output of the command is returned.
      return_stderr: If True, the standard error of the command is returned.
      expected_status: The expected return code. If not specified, defaults to
                       0. If the return code is a different value, an exception
                       is raised.
      stdin: A string of data to pipe to the process as standard input.

    Returns:
      A tuple containing the desired return values specified by the return_*
      arguments.
    """
    cmd = [gslib.GSUTIL_PATH] + ['--testexceptiontraces'] + cmd
    if IS_WINDOWS:
      cmd = [sys.executable] + cmd
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                         stdin=subprocess.PIPE)
    (stdout, stderr) = p.communicate(stdin)
    status = p.returncode

    if expected_status is not None:
      self.assertEqual(
          status, expected_status,
          msg='Expected status %d, got %d.\nCommand:\n%s\n\nstderr:\n%s' % (
              expected_status, status, ' '.join(cmd), stderr))

    toreturn = []
    if return_status:
      toreturn.append(status)
    if return_stdout:
      if IS_WINDOWS:
        stdout = stdout.replace('\r\n', '\n')
      toreturn.append(stdout)
    if return_stderr:
      if IS_WINDOWS:
        stderr = stderr.replace('\r\n', '\n')
      toreturn.append(stderr)

    if len(toreturn) == 1:
      return toreturn[0]
    elif toreturn:
      return tuple(toreturn)

  @contextmanager
  def SetAnonymousBotoCreds(self):
    boto_config_path = self.CreateTempFile(
        contents=BOTO_CONFIG_CONTENTS_IGNORE_ANON_WARNING)
    with SetBotoConfigForTest(boto_config_path):
      yield
<commit_msg>Fix bad merge of integration_testcase<commit_after># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains gsutil base integration test case class."""

from contextlib import contextmanager
import logging
import subprocess
import sys

import boto
from boto.exception import StorageResponseError
import gslib
from gslib.project_id import GOOG_PROJ_ID_HDR
from gslib.project_id import PopulateProjectId
from gslib.tests.testcase import base
import gslib.tests.util as util
from gslib.tests.util import RUN_S3_TESTS
from gslib.tests.util import SetBotoConfigForTest
from gslib.tests.util import unittest
from gslib.util import IS_WINDOWS
from gslib.util import Retry

LOGGER = logging.getLogger('integration-test')

# Contents of boto config file that will tell gsutil not to override the real
# error message with a warning about anonymous access if no credentials are
# provided in the config file.
BOTO_CONFIG_CONTENTS_IGNORE_ANON_WARNING = """
[Tests]
bypass_anonymous_access_warning = True
"""

def SkipForGS(reason):
  if not RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func

def SkipForS3(reason):
  if RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func

@unittest.skipUnless(util.RUN_INTEGRATION_TESTS,
                     'Not running integration tests.')
class GsUtilIntegrationTestCase(base.GsUtilTestCase):
  """Base class for gsutil integration tests."""
  GROUP_TEST_ADDRESS = 'gs-discussion@googlegroups.com'
  GROUP_TEST_ID = (
      '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5')
  USER_TEST_ADDRESS = 'gs-team@google.com'
  USER_TEST_ID = (
      '00b4903a9703325c6bfc98992d72e75600387a64b3b6bee9ef74613ef8842080')
  DOMAIN_TEST = 'google.com'
  # No one can create this bucket without owning the gmail.com domain, and we
  # won't create this bucket, so it shouldn't exist.
  # It would be nice to use google.com here but JSON API disallows
  # 'google' in resource IDs.
  nonexistent_bucket_name = 'nonexistent-bucket-foobar.gmail.com'

  def setUp(self):
    """Creates base configuration for integration tests."""
    super(GsUtilIntegrationTestCase, self).setUp()
    self.bucket_uris = []

    # Set up API version and project ID handler.
    self.api_version = boto.config.get_value(
        'GSUtil', 'default_api_version', '1')

    if util.RUN_S3_TESTS:
      self.test_api = 'XML'
      self.default_provider = 's3'
      self.nonexistent_bucket_name = (
          'nonexistentbucket-asf801rj3r9as90mfnnkjxpo02')
    else:
      # TODO: gsutil-beta: This should consider whether the command
      # supports the API or not, but since commands will default to XML
      # it is enough to assume XML support and override as needed in the tests.
      self.test_api = boto.config.get('GSUtil', 'force_api', 'JSON').upper()
      self.default_provider = 'gs'

  # Retry with an exponential backoff if a server error is received. This
  # ensures that we try *really* hard to clean up after ourselves.
  # TODO: As long as we're still using boto to do the teardown,
  # we decorate with boto exceptions.  Eventually this should be migrated
  # to CloudApi exceptions.
  @Retry(StorageResponseError, tries=6, timeout_secs=1)
  def tearDown(self):
    super(GsUtilIntegrationTestCase, self).tearDown()

    while self.bucket_uris:
      bucket_uri = self.bucket_uris[-1]
      try:
        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
      except StorageResponseError, e:
        # This can happen for tests of rm -r command, which for bucket-only
        # URIs delete the bucket at the end.
        if e.status == 404:
          self.bucket_uris.pop()
          continue
        else:
          raise
      while bucket_list:
        error = None
        for k in bucket_list:
          try:
            k.delete()
          except StorageResponseError, e:
            # This could happen if objects that have already been deleted are
            # still showing up in the listing due to eventual consistency. In
            # that case, we continue on until we've tried to deleted every
            # object in the listing before raising the error on which to retry.
            if e.status == 404:
              error = e
            else:
              raise
        if error:
          raise error  # pylint: disable=raising-bad-type
        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
      bucket_uri.delete_bucket()
      self.bucket_uris.pop()

  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
                   provider=None):
    """Creates a test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.
      storage_class: storage class to use. If not provided we us standard.
      provider: Provider to use - either "gs" (the default) or "s3".

    Returns:
      StorageUri for the created bucket.
    """
    if not provider:
      provider = self.default_provider
    bucket_name = bucket_name or self.MakeTempName('bucket')

    bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()),
                                  suppress_consec_slashes=False)

    if provider == 'gs':
      # Apply API version and project ID headers if necessary.
      headers = {'x-goog-api-version': self.api_version}
      headers[GOOG_PROJ_ID_HDR] = PopulateProjectId()
    else:
      headers = {}

    bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
    self.bucket_uris.append(bucket_uri)
    for i in range(test_objects):
      self.CreateObject(bucket_uri=bucket_uri,
                        object_name=self.MakeTempName('obj'),
                        contents='test %d' % i)
    return bucket_uri

  def CreateVersionedBucket(self, bucket_name=None, test_objects=0):
    """Creates a versioned test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.

    Returns:
      StorageUri for the created bucket with versioning enabled.
    """
    bucket_uri = self.CreateBucket(bucket_name=bucket_name,
                                   test_objects=test_objects)
    bucket_uri.configure_versioning(True)
    return bucket_uri

  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
    """Creates a test object.

    Args:
      bucket_uri: The URI of the bucket to place the object in. If not
                  specified, a new temporary bucket is created.
      object_name: The name to use for the object. If not specified, a temporary
                   test object name is constructed.
      contents: The contents to write to the object. If not specified, the key
                is not written to, which means that it isn't actually created
                yet on the server.

    Returns:
      A StorageUri for the created object.
    """
    bucket_uri = bucket_uri or self.CreateBucket()
    object_name = object_name or self.MakeTempName('obj')
    key_uri = bucket_uri.clone_replace_name(object_name)
    if contents is not None:
      key_uri.set_contents_from_string(contents)
    return key_uri

  def RunGsUtil(self, cmd, return_status=False, return_stdout=False,
                return_stderr=False, expected_status=0, stdin=None):
    """Runs the gsutil command.

    Args:
      cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
      return_status: If True, the exit status code is returned.
      return_stdout: If True, the standard output of the command is returned.
      return_stderr: If True, the standard error of the command is returned.
      expected_status: The expected return code. If not specified, defaults to
                       0. If the return code is a different value, an exception
                       is raised.
      stdin: A string of data to pipe to the process as standard input.

    Returns:
      A tuple containing the desired return values specified by the return_*
      arguments.
    """
    cmd = [gslib.GSUTIL_PATH] + ['--testexceptiontraces'] + cmd
    if IS_WINDOWS:
      cmd = [sys.executable] + cmd
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                         stdin=subprocess.PIPE)
    (stdout, stderr) = p.communicate(stdin)
    status = p.returncode

    if expected_status is not None:
      self.assertEqual(
          status, expected_status,
          msg='Expected status %d, got %d.\nCommand:\n%s\n\nstderr:\n%s' % (
              expected_status, status, ' '.join(cmd), stderr))

    toreturn = []
    if return_status:
      toreturn.append(status)
    if return_stdout:
      if IS_WINDOWS:
        stdout = stdout.replace('\r\n', '\n')
      toreturn.append(stdout)
    if return_stderr:
      if IS_WINDOWS:
        stderr = stderr.replace('\r\n', '\n')
      toreturn.append(stderr)

    if len(toreturn) == 1:
      return toreturn[0]
    elif toreturn:
      return tuple(toreturn)

  @contextmanager
  def SetAnonymousBotoCreds(self):
    boto_config_path = self.CreateTempFile(
        contents=BOTO_CONFIG_CONTENTS_IGNORE_ANON_WARNING)
    with SetBotoConfigForTest(boto_config_path):
      yield
<|endoftext|>


<commit_before>#!/usr/bin/python

# This code is original from jsmin by Douglas Crockford, it was translated to
# Python by Baruch Even. The original code had the following copyright and
# license.
#
# /* jsmin.c
#    2003-04-21
#
# Copyright (c) 2002 Douglas Crockford  (www.crockford.com)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# The Software shall be used for Good, not Evil.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# */

from StringIO import StringIO

def jsmin(js):
	ins = StringIO(js)
	outs = StringIO()
	JavascriptMinify().minify(ins, outs)
	str = outs.getvalue()
	if len(str) > 0 and str[0] == '\n':
		str = str[1:]
	return str

def isAlphanum(c):
	"""return true if the character is a letter, digit, underscore,
           dollar sign, or non-ASCII character.
	"""
	return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or
	        (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126));

class UnterminatedComment(Exception):
	pass

class UnterminatedStringLiteral(Exception):
	pass

class UnterminatedRegularExpression(Exception):
	pass

class JavascriptMinify(object):

	def _outA(self):
		self.outstream.write(self.theA)
	def _outB(self):
		self.outstream.write(self.theB)

	def _get(self):
		"""return the next character from stdin. Watch out for lookahead. If
		   the character is a control character, translate it to a space or
		   linefeed.
		"""
		c = self.theLookahead
		self.theLookahead = None
		if c == None:
			c = self.instream.read(1)
		if c >= ' ' or c == '\n':
			return c
		if c == '': # EOF
			return '\000'
		if c == '\r':
			return '\n'
		return ' '

	def _peek(self):
		self.theLookahead = self._get()
		return self.theLookahead

	def _next(self):
		"""get the next character, excluding comments. peek() is used to see
		   if a '/' is followed by a '/' or '*'.
		"""
		c = self._get()
		if c == '/':
			p = self._peek()
			if p == '/':
				c = self._get()
				while c > '\n':
					c = self._get()
				return c
			if p == '*':
				c = self._get()
				while 1:
					c = self._get()
					if c == '*':
						if self._peek() == '/':
							self._get()
							return ' '
					if c == '\000':
						raise UnterminatedComment()

		return c

	def _action(self, action):
		"""do something! What you do is determined by the argument:
		   1   Output A. Copy B to A. Get the next B.
		   2   Copy B to A. Get the next B. (Delete A).
		   3   Get the next B. (Delete B).
		   action treats a string as a single character. Wow!
		   action recognizes a regular expression if it is preceded by ( or , or =.
		"""
		if action <= 1:
			self._outA()

		if action <= 2:
			self.theA = self.theB
			if self.theA == "'" or self.theA == '"':
				while 1:
					self._outA()
					self.theA = self._get()
					if self.theA == self.theB:
						break
					if self.theA <= '\n':
						raise UnterminatedStringLiteral()
					if self.theA == '\\':
						self._outA()
						self.theA = self._get()

		if action <= 3:
			self.theB = self._next()
			if self.theB == '/' and (self.theA == '(' or self.theA == ',' or self.theA == '='):
				self._outA()
				self._outB()
				while 1:
					self.theA = self._get()
					if self.theA == '/':
						break
					elif self.theA == '\\':
						self._outA()
						self.theA = self._get()
					elif self.theA <= '\n':
						raise UnterminatedRegularExpression()
					self._outA()
				self.theB = self._next()

	def _jsmin(self):
		"""Copy the input to the output, deleting the characters which are
		   insignificant to JavaScript. Comments will be removed. Tabs will be
		   replaced with spaces. Carriage returns will be replaced with linefeeds.
		   Most spaces and linefeeds will be removed.
		"""
		self.theA = '\n'
		self._action(3)

		while self.theA != '\000':
			if self.theA == ' ':
				if isAlphanum(self.theB):
					self._action(1)
				else:
					self._action(2)
			elif self.theA == '\n':
				if self.theB in ['{', '[', '(', '+', '-']:
					self._action(1)
				elif self.theB == ' ':
					self._action(3)
				else:
					if isAlphanum(self.theB):
						self._action(1)
					else:
						self._action(2)
			else:
				if self.theB == ' ':
					if isAlphanum(self.theA):
						self._action(1)
					else:
						self._action(3)
				elif self.theB == '\n':
					if self.theA in ['}', ']', ')', '+', '-', '"', '\'']:
						self._action(1)
					else:
						if isAlphanum(self.theA):
							self._action(1)
						else:
							self._action(3)
				else:
					self._action(1)

	def minify(self, instream, outstream):
		self.instream = instream
		self.outstream = outstream
		self.theA = None
		self.thaB = None
		self.theLookahead = None

		self._jsmin()
		self.instream.close()

if __name__ == '__main__':
	import sys
	jsm = JavascriptMinify()
	jsm.minify(sys.stdin, sys.stdout)
<commit_msg>Set jsmin.py executable.<commit_after>#!/usr/bin/python

# This code is original from jsmin by Douglas Crockford, it was translated to
# Python by Baruch Even. The original code had the following copyright and
# license.
#
# /* jsmin.c
#    2003-04-21
#
# Copyright (c) 2002 Douglas Crockford  (www.crockford.com)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# The Software shall be used for Good, not Evil.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# */

from StringIO import StringIO

def jsmin(js):
	ins = StringIO(js)
	outs = StringIO()
	JavascriptMinify().minify(ins, outs)
	str = outs.getvalue()
	if len(str) > 0 and str[0] == '\n':
		str = str[1:]
	return str

def isAlphanum(c):
	"""return true if the character is a letter, digit, underscore,
           dollar sign, or non-ASCII character.
	"""
	return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or
	        (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126));

class UnterminatedComment(Exception):
	pass

class UnterminatedStringLiteral(Exception):
	pass

class UnterminatedRegularExpression(Exception):
	pass

class JavascriptMinify(object):

	def _outA(self):
		self.outstream.write(self.theA)
	def _outB(self):
		self.outstream.write(self.theB)

	def _get(self):
		"""return the next character from stdin. Watch out for lookahead. If
		   the character is a control character, translate it to a space or
		   linefeed.
		"""
		c = self.theLookahead
		self.theLookahead = None
		if c == None:
			c = self.instream.read(1)
		if c >= ' ' or c == '\n':
			return c
		if c == '': # EOF
			return '\000'
		if c == '\r':
			return '\n'
		return ' '

	def _peek(self):
		self.theLookahead = self._get()
		return self.theLookahead

	def _next(self):
		"""get the next character, excluding comments. peek() is used to see
		   if a '/' is followed by a '/' or '*'.
		"""
		c = self._get()
		if c == '/':
			p = self._peek()
			if p == '/':
				c = self._get()
				while c > '\n':
					c = self._get()
				return c
			if p == '*':
				c = self._get()
				while 1:
					c = self._get()
					if c == '*':
						if self._peek() == '/':
							self._get()
							return ' '
					if c == '\000':
						raise UnterminatedComment()

		return c

	def _action(self, action):
		"""do something! What you do is determined by the argument:
		   1   Output A. Copy B to A. Get the next B.
		   2   Copy B to A. Get the next B. (Delete A).
		   3   Get the next B. (Delete B).
		   action treats a string as a single character. Wow!
		   action recognizes a regular expression if it is preceded by ( or , or =.
		"""
		if action <= 1:
			self._outA()

		if action <= 2:
			self.theA = self.theB
			if self.theA == "'" or self.theA == '"':
				while 1:
					self._outA()
					self.theA = self._get()
					if self.theA == self.theB:
						break
					if self.theA <= '\n':
						raise UnterminatedStringLiteral()
					if self.theA == '\\':
						self._outA()
						self.theA = self._get()

		if action <= 3:
			self.theB = self._next()
			if self.theB == '/' and (self.theA == '(' or self.theA == ',' or self.theA == '='):
				self._outA()
				self._outB()
				while 1:
					self.theA = self._get()
					if self.theA == '/':
						break
					elif self.theA == '\\':
						self._outA()
						self.theA = self._get()
					elif self.theA <= '\n':
						raise UnterminatedRegularExpression()
					self._outA()
				self.theB = self._next()

	def _jsmin(self):
		"""Copy the input to the output, deleting the characters which are
		   insignificant to JavaScript. Comments will be removed. Tabs will be
		   replaced with spaces. Carriage returns will be replaced with linefeeds.
		   Most spaces and linefeeds will be removed.
		"""
		self.theA = '\n'
		self._action(3)

		while self.theA != '\000':
			if self.theA == ' ':
				if isAlphanum(self.theB):
					self._action(1)
				else:
					self._action(2)
			elif self.theA == '\n':
				if self.theB in ['{', '[', '(', '+', '-']:
					self._action(1)
				elif self.theB == ' ':
					self._action(3)
				else:
					if isAlphanum(self.theB):
						self._action(1)
					else:
						self._action(2)
			else:
				if self.theB == ' ':
					if isAlphanum(self.theA):
						self._action(1)
					else:
						self._action(3)
				elif self.theB == '\n':
					if self.theA in ['}', ']', ')', '+', '-', '"', '\'']:
						self._action(1)
					else:
						if isAlphanum(self.theA):
							self._action(1)
						else:
							self._action(3)
				else:
					self._action(1)

	def minify(self, instream, outstream):
		self.instream = instream
		self.outstream = outstream
		self.theA = None
		self.thaB = None
		self.theLookahead = None

		self._jsmin()
		self.instream.close()

if __name__ == '__main__':
	import sys
	jsm = JavascriptMinify()
	jsm.minify(sys.stdin, sys.stdout)
<|endoftext|>


<commit_before># Pauli operators
#
# A Pauli Opt.
#
# Jay Gambetta <jay.gambetta@us.ibm.com>
# Andrew Cross <awcross@us.ibm.com>

import random
import numpy as np

class Pauli:
    """A simple class representing Pauli Operators.

    The form is P = (-i)^dot(v,w) Z^v X^w where v and w are elements of Z_2^n.
    That is, there are 4^n elements (no phases in this group).

    For example, for 1 qubit
    P_00 = Z^0 X^0 = I
    P_01 = X
    P_10 = Z
    P_11 = -iZX = (-i) iY = Y

    Multiplication is P1*P2 = (-i)^dot(v1+v2,w1+w2) Z^(v1+v2) X^(w1+w2)
    where the sums are taken modulo 2.
    """

    def __init__(self, v, w):
        """Make the Pauli class."""
        self.numberofqubits = len(v)
        self.v = v
        self.w = w

    def __str__(self):
        """Output the Pauli as first row v and second row w."""
        stemp = 'v = '
        for i in self.v:
            stemp += str(i) + '\t'
        stemp = stemp + '\nw = '
        for j in self.w:
            stemp += str(j) + '\t'
        return stemp

    def __mul__(self, other):
        """Multiply two Paulis."""
        if self.numberofqubits != other.numberofqubits:
            print('Paulis cannot be multiplied - different number of qubits')
        vnew = (self.v + other.v) % 2
        wnew = (self.w + other.w) % 2
        paulinew = Pauli(vnew, wnew)
        return paulinew

    def to_label(self):
        """Print out the labels in X, Y, Z format."""
        plabel = ''
        for jindex in range(self.numberofqubits):
            if self.v[jindex] == 0 and self.w[jindex] == 0:
                plabel += 'I'
            elif self.v[jindex] == 0 and self.w[jindex] == 1:
                plabel += 'X'
            elif self.v[jindex] == 1 and self.w[jindex] == 1:
                plabel += 'Y'
            elif self.v[jindex] == 1 and self.w[jindex] == 0:
                plabel += 'Z'
        return plabel

    def to_matrix(self):
        """Convert Pauli to a matrix representation.

        Order is q_n x q_{n-1} .... q_0
        """
        X = np.array([[0, 1], [1, 0]], dtype=complex)
        Z = np.array([[1, 0], [0, -1]], dtype=complex)
        Id = np.array([[1, 0], [0, 1]], dtype=complex)
        Xtemp = 1
        for k in range(self.numberofqubits):
            if self.v[k] == 0:
                tempz = Id
            elif self.v[k] == 1:
                tempz = Z
            else:
                print('the z string is not of the form 0 and 1')
            if self.w[k] == 0:
                tempx = Id
            elif self.w[k] == 1:
                tempx = X
            else:
                print('the x string is not of the form 0 and 1')
            ope = np.dot(tempz, tempx)
            Xtemp = np.kron(ope, Xtemp)
        pauli_mat = (-1j)**np.dot(self.v, self.w) * Xtemp
        return pauli_mat

def random_pauli(numberofqubits):
    """Return a random Pauli on numberofqubits."""
    v = np.array(list(bin(random.getrandbits(numberofqubits))
                      [2:].zfill(numberofqubits))).astype(np.int)
    w = np.array(list(bin(random.getrandbits(numberofqubits))
                      [2:].zfill(numberofqubits))).astype(np.int)
    return Pauli(v, w)

def sgn_prod(P1, P2):
    """Multiply two Paulis P1*P2 and track the sign"""

    if P1.numberofqubits != P2.numberofqubits:
        print('Paulis cannot be multiplied - different number of qubits')
    vnew = (P1.v + P2.v) % 2
    wnew = (P1.w + P2.w) % 2
    paulinew = Pauli(vnew, wnew)
    sgn = np.inner(P1.w, P2.v) % 2
    return paulinew, sgn

def inverse_pauli(other):
    """Return the inverse of a Pauli."""
    v = other.v
    w = other.w
    return Pauli(v, w)

def label_to_pauli(label):
    """Return the pauli of a string ."""
    v = np.zeros(len(label))
    w = np.zeros(len(label))
    for j in range(len(label)):
        if label[j] == 'I':
            v[j] = 0
            w[j] = 0
        elif label[j] == 'Z':
            v[j] = 1
            w[j] = 0
        elif label[j] == 'Y':
            v[j] = 1
            w[j] = 1
        elif label[j] == 'X':
            v[j] = 0
            w[j] = 1
        else:
            print('something went wrong')
            return -1
    return Pauli(v, w)

def pauli_group(numberofqubits, case=0):
    """Return the Pauli group with 4^n elements.

    The phases have been removed.
    case 0 is ordered by Pauli weights and
    case 1 is ordered by I,X,Y,Z counting last qubit fastest.
    @param numberofqubits is number of qubits
    @param case determines ordering of group elements (0=weight, 1=tensor)
    @return list of Pauli objects
    WARNING THIS IS EXPONENTIAL
    """
    if numberofqubits < 5:
        tempset = []
        if case == 0:
            tmp = pauli_group(numberofqubits, case=1)
            # sort on the weight of the Pauli operator
            return sorted(tmp, key=lambda x: -
                          np.count_nonzero(np.array(x.to_label(), 'c') == b'I'))

        elif case == 1:
            # the Pauli set is in tensor order II IX IY IZ XI ...
            for kindex in range(4**numberofqubits):
                v = np.zeros(numberofqubits)
                w = np.zeros(numberofqubits)
                # looping over all the qubits
                for jindex in range(numberofqubits):
                    # making the Pauli for each kindex i fill it in from the
                    # end first
                    element = int((kindex) / (4**(jindex))) % 4
                    if element == 0:
                        v[jindex] = 0
                        w[jindex] = 0
                    elif element == 1:
                        v[jindex] = 0
                        w[jindex] = 1
                    elif element == 2:
                        v[jindex] = 1
                        w[jindex] = 1
                    elif element == 3:
                        v[jindex] = 1
                        w[jindex] = 0
                tempset.append(Pauli(v, w))
            return tempset
    else:
        print('please set the number of qubits to less than 5')
        return -1
<commit_msg>track phase in product of pauli<commit_after># Pauli operators
#
# A Pauli Opt.
#
# Jay Gambetta <jay.gambetta@us.ibm.com>
# Andrew Cross <awcross@us.ibm.com>

import random
import numpy as np

class Pauli:
    """A simple class representing Pauli Operators.

    The form is P = (-i)^dot(v,w) Z^v X^w where v and w are elements of Z_2^n.
    That is, there are 4^n elements (no phases in this group).

    For example, for 1 qubit
    P_00 = Z^0 X^0 = I
    P_01 = X
    P_10 = Z
    P_11 = -iZX = (-i) iY = Y

    Multiplication is P1*P2 = (-i)^dot(v1+v2,w1+w2) Z^(v1+v2) X^(w1+w2)
    where the sums are taken modulo 2.
    """

    def __init__(self, v, w):
        """Make the Pauli class."""
        self.numberofqubits = len(v)
        self.v = v
        self.w = w

    def __str__(self):
        """Output the Pauli as first row v and second row w."""
        stemp = 'v = '
        for i in self.v:
            stemp += str(i) + '\t'
        stemp = stemp + '\nw = '
        for j in self.w:
            stemp += str(j) + '\t'
        return stemp

    def __mul__(self, other):
        """Multiply two Paulis."""
        if self.numberofqubits != other.numberofqubits:
            print('Paulis cannot be multiplied - different number of qubits')
        vnew = (self.v + other.v) % 2
        wnew = (self.w + other.w) % 2
        paulinew = Pauli(vnew, wnew)
        return paulinew

    def to_label(self):
        """Print out the labels in X, Y, Z format."""
        plabel = ''
        for jindex in range(self.numberofqubits):
            if self.v[jindex] == 0 and self.w[jindex] == 0:
                plabel += 'I'
            elif self.v[jindex] == 0 and self.w[jindex] == 1:
                plabel += 'X'
            elif self.v[jindex] == 1 and self.w[jindex] == 1:
                plabel += 'Y'
            elif self.v[jindex] == 1 and self.w[jindex] == 0:
                plabel += 'Z'
        return plabel

    def to_matrix(self):
        """Convert Pauli to a matrix representation.

        Order is q_n x q_{n-1} .... q_0
        """
        X = np.array([[0, 1], [1, 0]], dtype=complex)
        Z = np.array([[1, 0], [0, -1]], dtype=complex)
        Id = np.array([[1, 0], [0, 1]], dtype=complex)
        Xtemp = 1
        for k in range(self.numberofqubits):
            if self.v[k] == 0:
                tempz = Id
            elif self.v[k] == 1:
                tempz = Z
            else:
                print('the z string is not of the form 0 and 1')
            if self.w[k] == 0:
                tempx = Id
            elif self.w[k] == 1:
                tempx = X
            else:
                print('the x string is not of the form 0 and 1')
            ope = np.dot(tempz, tempx)
            Xtemp = np.kron(ope, Xtemp)
        pauli_mat = (-1j)**np.dot(self.v, self.w) * Xtemp
        return pauli_mat

def random_pauli(numberofqubits):
    """Return a random Pauli on numberofqubits."""
    v = np.array(list(bin(random.getrandbits(numberofqubits))
                      [2:].zfill(numberofqubits))).astype(np.int)
    w = np.array(list(bin(random.getrandbits(numberofqubits))
                      [2:].zfill(numberofqubits))).astype(np.int)
    return Pauli(v, w)

def sgn_prod(P1, P2):
    """Multiply two Paulis P1*P2 and track the phase"""

    if P1.numberofqubits != P2.numberofqubits:
        print('Paulis cannot be multiplied - different number of qubits')
    vnew = (P1.v + P2.v) % 2
    wnew = (P1.w + P2.w) % 2
    paulinew = Pauli(vnew, wnew)
    phase=1
    for i in range(len(P1.v)):
        if P1.v[i]==1 and P1.w[i]==0 and P2.v[i]==0 and P2.w[i]==1:  # Z*X
            phase=1j*phase
        if P1.v[i]==0 and P1.w[i]==1 and P2.v[i]==1 and P2.w[i]==0:  # X*Z
            phase=-1j*phase
        if P1.v[i]==0 and P1.w[i]==1 and P2.v[i]==1 and P2.w[i]==1:  # X*Y
            phase=1j*phase
        if P1.v[i]==1 and P1.w[i]==1 and P2.v[i]==0 and P2.w[i]==1:  # Y*X
            phase=-1j*phase
        if P1.v[i]==1 and P1.w[i]==1 and P2.v[i]==1 and P2.w[i]==0:  # Y*Z
            phase=1j*phase
        if P1.v[i]==1 and P1.w[i]==0 and P2.v[i]==1 and P2.w[i]==1:  # Z*Y
            phase=-1j*phase

    return paulinew, phase

def inverse_pauli(other):
    """Return the inverse of a Pauli."""
    v = other.v
    w = other.w
    return Pauli(v, w)

def label_to_pauli(label):
    """Return the pauli of a string ."""
    v = np.zeros(len(label))
    w = np.zeros(len(label))
    for j in range(len(label)):
        if label[j] == 'I':
            v[j] = 0
            w[j] = 0
        elif label[j] == 'Z':
            v[j] = 1
            w[j] = 0
        elif label[j] == 'Y':
            v[j] = 1
            w[j] = 1
        elif label[j] == 'X':
            v[j] = 0
            w[j] = 1
        else:
            print('something went wrong')
            return -1
    return Pauli(v, w)

def pauli_group(numberofqubits, case=0):
    """Return the Pauli group with 4^n elements.

    The phases have been removed.
    case 0 is ordered by Pauli weights and
    case 1 is ordered by I,X,Y,Z counting last qubit fastest.
    @param numberofqubits is number of qubits
    @param case determines ordering of group elements (0=weight, 1=tensor)
    @return list of Pauli objects
    WARNING THIS IS EXPONENTIAL
    """
    if numberofqubits < 5:
        tempset = []
        if case == 0:
            tmp = pauli_group(numberofqubits, case=1)
            # sort on the weight of the Pauli operator
            return sorted(tmp, key=lambda x: -
                          np.count_nonzero(np.array(x.to_label(), 'c') == b'I'))

        elif case == 1:
            # the Pauli set is in tensor order II IX IY IZ XI ...
            for kindex in range(4**numberofqubits):
                v = np.zeros(numberofqubits)
                w = np.zeros(numberofqubits)
                # looping over all the qubits
                for jindex in range(numberofqubits):
                    # making the Pauli for each kindex i fill it in from the
                    # end first
                    element = int((kindex) / (4**(jindex))) % 4
                    if element == 0:
                        v[jindex] = 0
                        w[jindex] = 0
                    elif element == 1:
                        v[jindex] = 0
                        w[jindex] = 1
                    elif element == 2:
                        v[jindex] = 1
                        w[jindex] = 1
                    elif element == 3:
                        v[jindex] = 1
                        w[jindex] = 0
                tempset.append(Pauli(v, w))
            return tempset
    else:
        print('please set the number of qubits to less than 5')
        return -1
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
# sys.path.insert(1, "./libs")
#from utils.perf_analysis import PerfAnalysis
#from utils.trace_analysis import TraceAnalysis
from perf_analysis import PerfAnalysis
from trace import Trace
from trace_analysis import TraceAnalysis

import os
import re
import argparse
import json

# Configure logging
import logging
reload(logging)
logging.basicConfig(
    format='%(asctime)-9s %(levelname)-8s: %(message)s',
    level=logging.DEBUG,
    # level=logging.INFO,
    datefmt='%I:%M:%S')

# Regexp to match the format of a result folder
TEST_DIR_RE = re.compile(
        r'([^:]*):([^:]*):([^:]*)'
    )

parser = argparse.ArgumentParser(
        description='EAS Performance and Trace Plotter')
parser.add_argument('--results', type=str,
        default='./results_latest',
        help='Folder containing experimental results')
parser.add_argument('--outdir', type=str,
        default=None,
        help='A single output folder we want to produce plots for')
parser.add_argument('--tmin', type=float,
        default=None,
        help='Minimum timestamp for all plots')
parser.add_argument('--tmax', type=float,
        default=None,
        help='Maximum timestamp for all plots')
parser.add_argument('--plots', type=str,
        default='all',
        help='List of plots to produce (all,')

args = None

def main():
    global args
    args = parser.parse_args()

    # Setup plots to produce
    if args.plots == 'all':
        args.plots = 'tasks clusters cpus stune ediff edspace'

    # For each rtapp and each run
    if args.outdir is not None:

        # Load platform descriptior
        platform = None
        plt_file = os.path.join(args.outdir, 'platform.json')
        if os.path.isfile(plt_file):
            with open(plt_file, 'r') as ifile:
                platform = json.load(ifile)
        logging.info('Platform description:')
        logging.info('  %s', platform)

        # Plot the specified results folder
        return plotdir(args.outdir, platform)

    for test_idx in sorted(os.listdir(args.results)):

        match = TEST_DIR_RE.search(test_idx)
        if match == None:
            continue
        wtype = match.group(1)
        conf_idx = match.group(2)
        wload_idx = match.group(3)

        # Generate performance plots only for RTApp workloads
        if wtype != 'rtapp':
            continue

        logging.debug('Processing [%s:%s:%s]...',
                wtype, conf_idx, wload_idx)

        # For each run of an rt-app workload
        test_dir = os.path.join(args.results, test_idx)

        # Load platform descriptior
        platform = None
        plt_file = os.path.join(test_dir, 'platform.json')
        if os.path.isfile(plt_file):
            with open(plt_file, 'r') as ifile:
                platform = json.load(ifile)
        logging.info('Platform description:')
        logging.info('  %s', platform)

        for run_idx in sorted(os.listdir(test_dir)):

            run_dir = os.path.join(test_dir, run_idx)
            try:
                run_id = int(run_idx)
            except ValueError:
                continue

            logging.info('Generate plots for [%s]...', run_dir)
            plotdir(run_dir, platform)

def plotdir(run_dir, platform):
    global args
    tasks = None
    pa = None

    # Load RTApp performance data
    try:
        pa = PerfAnalysis(run_dir)

        # Get the list of RTApp tasks
        tasks = pa.tasks()
        logging.info('Tasks: %s', tasks)
    except ValueError:
        pa = None
        logging.info('No performance data found')

    # Load Trace Analysis modules
    trace = Trace(platform, run_dir, tasks)
    ta = TraceAnalysis(trace, tasks)

    # Define time ranges for all the temporal plots
    ta.setXTimeRange(args.tmin, args.tmax)

    # Tasks plots
    if 'tasks' in args.plots:
        ta.plotTasks()
        if pa:
            for task in tasks:
                pa.plotPerf(task)

    # Cluster and CPUs plots
    if 'clusters' in args.plots:
        ta.plotClusterFrequencies()
    if 'cpus' in args.plots:
        ta.plotCPU()

    # SchedTune plots
    if 'stune' in args.plots:
        ta.plotSchedTuneConf()
    if 'ediff' in args.plots:
        ta.plotEDiffTime();
    if 'edspace' in args.plots:
        ta.plotEDiffSpace();

if __name__ == "__main__":
    main()
<commit_msg>tools/plots: replace TraceAnalysis with refactored analysis API<commit_after>#!/usr/bin/python
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
# sys.path.insert(1, "./libs")

from perf_analysis import PerfAnalysis
from trace import Trace

import os
import re
import argparse
import json

# Configure logging
import logging
reload(logging)
logging.basicConfig(
    format='%(asctime)-9s %(levelname)-8s: %(message)s',
    level=logging.DEBUG,
    # level=logging.INFO,
    datefmt='%I:%M:%S')

# Regexp to match the format of a result folder
TEST_DIR_RE = re.compile(
        r'([^:]*):([^:]*):([^:]*)'
    )

parser = argparse.ArgumentParser(
        description='EAS Performance and Trace Plotter')
parser.add_argument('--results', type=str,
        default='./results_latest',
        help='Folder containing experimental results')
parser.add_argument('--outdir', type=str,
        default=None,
        help='A single output folder we want to produce plots for')
parser.add_argument('--tmin', type=float,
        default=None,
        help='Minimum timestamp for all plots')
parser.add_argument('--tmax', type=float,
        default=None,
        help='Maximum timestamp for all plots')
parser.add_argument('--plots', type=str,
        default='all',
        help='List of plots to produce (all,')

args = None

def main():
    global args
    args = parser.parse_args()

    # Setup plots to produce
    if args.plots == 'all':
        args.plots = 'tasks clusters cpus stune ediff edspace'

    # For each rtapp and each run
    if args.outdir is not None:

        # Load platform descriptior
        platform = None
        plt_file = os.path.join(args.outdir, 'platform.json')
        if os.path.isfile(plt_file):
            with open(plt_file, 'r') as ifile:
                platform = json.load(ifile)
        logging.info('Platform description:')
        logging.info('  %s', platform)

        # Plot the specified results folder
        return plotdir(args.outdir, platform)

    for test_idx in sorted(os.listdir(args.results)):

        match = TEST_DIR_RE.search(test_idx)
        if match == None:
            continue
        wtype = match.group(1)
        conf_idx = match.group(2)
        wload_idx = match.group(3)

        # Generate performance plots only for RTApp workloads
        if wtype != 'rtapp':
            continue

        logging.debug('Processing [%s:%s:%s]...',
                wtype, conf_idx, wload_idx)

        # For each run of an rt-app workload
        test_dir = os.path.join(args.results, test_idx)

        # Load platform descriptior
        platform = None
        plt_file = os.path.join(test_dir, 'platform.json')
        if os.path.isfile(plt_file):
            with open(plt_file, 'r') as ifile:
                platform = json.load(ifile)
        logging.info('Platform description:')
        logging.info('  %s', platform)

        for run_idx in sorted(os.listdir(test_dir)):

            run_dir = os.path.join(test_dir, run_idx)
            try:
                run_id = int(run_idx)
            except ValueError:
                continue

            logging.info('Generate plots for [%s]...', run_dir)
            plotdir(run_dir, platform)

def plotdir(run_dir, platform):
    global args
    tasks = None
    pa = None

    # Load RTApp performance data
    try:
        pa = PerfAnalysis(run_dir)

        # Get the list of RTApp tasks
        tasks = pa.tasks()
        logging.info('Tasks: %s', tasks)
    except ValueError:
        pa = None
        logging.info('No performance data found')

    # Load Trace Analysis modules
    trace = Trace(platform, run_dir, tasks)

    # Define time ranges for all the temporal plots
    trace.setXTimeRange(args.tmin, args.tmax)

    # Tasks plots
    if 'tasks' in args.plots:
        trace.analysis.tasks.plotTasks()
        if pa:
            for task in tasks:
                pa.plotPerf(task)

    # Cluster and CPUs plots
    if 'clusters' in args.plots:
        trace.analysis.frequency.plotClusterFrequencies()
    if 'cpus' in args.plots:
        trace.analysis.cpus.plotCPU()

    # SchedTune plots
    if 'stune' in args.plots:
        trace.analysis.eas.plotSchedTuneConf()
    if 'ediff' in args.plots:
        trace.analysis.eas.plotEDiffTime();
    if 'edspace' in args.plots:
        trace.analysis.eas.plotEDiffSpace();

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys, os, json, re, time
import getpass
import subprocess
import readline
import argparse
import shutil

if sys.version_info <= (3, 3):
    print("This script requires Python 3.4 or higher")
    sys.exit(-1)

dopip = False
try:
    from elasticsearch import Elasticsearch
    from formatflowed import convertToWrapped
    from netaddr import IPNetwork
except:
    dopip = True

if dopip and (getpass.getuser() != "root"):
    print("It looks like you need to install some python modules first")
    print("Either run this as root to do so, or run: ")
    print("pip3 install elasticsearch formatflowed netaddr")
    sys.exit(-1)

elif dopip:
    print("Before we get started, we need to install some modules")
    print("Hang on!")
    try:
        subprocess.check_call(('pip3','install','elasticsearch','formatflowed', 'netaddr'))
        from elasticsearch import Elasticsearch
        from formatflowed import convertToWrapped
        from netaddr import IPNetwork
    except:
        print("Oh dear, looks like this failed :(")
        print("Please install elasticsearch and formatflowed before you try again:")
        print("pip install elasticsearch formatflowed netaddr")
        sys.exit(-1)

# CLI arg parsing
parser = argparse.ArgumentParser(description='Command line options.')

parser.add_argument('--defaults', dest='defaults', action='store_true',
                   help='Use default settings')

parser.add_argument('--dbhost', dest='dbhost', type=str, nargs=1,
                   help='ES backend hostname')
parser.add_argument('--dbport', dest='dbport', type=str, nargs=1,
                   help='DB port')
parser.add_argument('--dbname', dest='dbname', type=str, nargs=1,
                   help='ES DB name')
parser.add_argument('--mailserver', dest='mailserver', type=str, nargs=1,
                   help='Host name of outgoing mail server')
parser.add_argument('--mldom', dest='mldom', type=str, nargs=1,
                   help='Domains to accept mail for via UI')
parser.add_argument('--wordcloud', dest='wc', action='store_true',
                   help='Enable word cloud')
parser.add_argument('--skiponexist', dest='soe', action='store_true',
                   help='Skip setup if ES index exists')
parser.add_argument('--noindex', dest='noi', action='store_true',
                   help="Don't make an ES index, assume it exists")
parser.add_argument('--nocloud', dest='nwc', action='store_true',
                   help='Do not enable word cloud')
args = parser.parse_args()

print("Welcome to the Pony Mail setup script!")
print("Let's start by determining some settings...")
print("")

hostname = ""
port = 0
dbname = ""
mlserver = ""
mldom = ""
wc = ""
wce = False

# If called with --defaults (like from Docker), use default values
if args.defaults:
    hostname = "localhost"
    port = 9200
    dbname = "ponymail"
    mlserver = "localhost"
    mldom = "example.org"
    wc = "Y"
    wce = True

# Accept CLI args, copy them
if args.dbhost:
    hostname = args.dbhost[0]
if args.dbport:
    port = int(args.dbport[0])
if args.dbname:
    dbname = args.dbname[0]
if args.mailserver:
    mlserver = args.mailserver[0]
if args.mldom:
    mldom = args.mldom[0]
if args.wc:
    wc = args.wc
if args.nwc:
    wc = False

while hostname == "":
    hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ")

while port < 1:
    try:
        port = int(input("What port is ElasticSearch listening on? (normally 9200): "))
    except ValueError:
        pass

while dbname == "":
    dbname = input("What would you like to call the mail index (e.g. ponymail): ")

while mlserver == "":
    mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ")

while mldom == "":
    mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ")

while wc == "":
    wc = input("Would you like to enable the word cloud feature? (Y/N): ")
    if wc.lower() == "y":
        wce = True

print("Okay, I got all I need, setting up Pony Mail...")

if not args.noi:
    print("Creating index " + dbname)

    es = Elasticsearch([
        {
            'host': hostname,
            'port': port,
            'use_ssl': False,
            'url_prefix': ''
        }],
        max_retries=5,
        retry_on_timeout=True
        )
    mappings = {
        "mbox" : {
          "properties" : {
            "@import_timestamp" : {
              "type" : "date",
              "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
            },
            "@version" : {
              "type" : "long"
            },
            "attachments" : {
              "properties" : {
                "content_type" : {
                  "type" : "string"
                },
                "filename" : {
                  "type" : "string"
                },
                "hash" : {
                  "type" : "string"
                },
                "size" : {
                  "type" : "long"
                }
              }
            },
            "body" : {
              "type" : "string"
            },
            "cc": {
              "type": "string"
            },
            "date" : {
              "type" : "date",
              "store" : True,
              "format" : "yyyy/MM/dd HH:mm:ss",
              "index" : "not_analyzed"
            },
            "epoch" : {
              "type" : "double",
              "index" : "not_analyzed"
            },
            "from" : {
              "type" : "string"
            },
            "from_raw" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "in-reply-to" : {
              "type" : "string"
            },
            "list" : {
              "type" : "string"
            },
            "list_raw" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "message-id" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "mid" : {
              "type" : "string"
            },
            "private" : {
              "type" : "boolean"
            },
            "references" : {
              "type" : "string"
            },
            "subject" : {
              "type" : "string"
            },
            "to" : {
              "type" : "string"
            }
          }
        },
        "attachment" : {
          "properties" : {
            "source" : {
              "type" : "binary"
            }
          }
        },
        "mbox_source" : {
          "properties" : {
            "source" : {
              "type" : "binary"
            },
            "@version" : {
              "type" : "long"
            },
            "message-id" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "mid" : {
              "type" : "string"
            }
          }
        },
        "mailinglists" : {
          "properties" : {
            "description" : {
              "type" : "string"
            },
            "list" : {
              "type" : "string"
            },
            "name" : {
              "type" : "string"
            }
          }
        }
    }

    # Check if index already exists
    if es.indices.exists(dbname):
        if args.soe:
            print("Index already exists and SOE set, exiting quietly")
            sys.exit(0)
        else:
            print("Error: ElasticSearch index %s already exists!" % dbname)
            sys.exit(-1)

    res = es.indices.create(index = dbname, body = {
                "mappings" : mappings
            }
        )

    print("Index created!")

print("Writing importer config (ponymail.cfg)")

with open("ponymail.cfg", "w") as f:
    f.write("""
###############################################################
# Pony Mail Configuration file

# Main ES configuration
[elasticsearch]
hostname:               %s
dbname:                 %s
port:                   %u
ssl:                    false

#uri:                   url_prefix

#user:                  username
#password:              password

#write:                 consistency level (default quorum)

#backup:                database name

[archiver]
#generator:             medium|full|other

[debug]
#cropout:               string to crop from list-id

###############################################################
            """ % (hostname, dbname, port))
    f.close()

print("mod_lua configuration (config.lua)")
with open("../site/api/lib/config.lua", "w") as f:
    f.write("""
local config = {
    es_url = "http://%s:%u/%s/",
    mailserver = "%s",
    accepted_domains = "%s",
    wordcloud = %s,
    slow_count = false,
    email_footer = nil, -- see the docs for how to set this up.
    full_headers = false,
    maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists
    admin_oauth = {}, -- list of domains that may do administrative oauth (private list access)
                     -- add 'www.googleapis.com' to the list for google oauth to decide, for instance.
    oauth_fields = {}, -- used for specifying individual oauth handling parameters.
    antispam = true  -- Whether or not to add anti-spam measures aimed at anonymous users.
    -- no_association = {} -- domains that are not allowed for email association
    -- hidePrivate = true -- whether to hide names of lists that contain any private mails
    -- listsDisplay = 'regex' -- if defined, hide list names that don't match the regex
}
return config
            """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
    f.close()

print("Copying sample JS config to config.js (if needed)...")
if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"):
    shutil.copy("../site/js/config.js.sample", "../site/js/config.js")

print("All done, Pony Mail should...work now :)")
print("If you are using an external mail inbound server, \nmake sure to copy archiver.py and ponymail.cfg to it")
<commit_msg>More missing doc types<commit_after>#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys, os, json, re, time
import getpass
import subprocess
import readline
import argparse
import shutil

if sys.version_info <= (3, 3):
    print("This script requires Python 3.4 or higher")
    sys.exit(-1)

dopip = False
try:
    from elasticsearch import Elasticsearch
    from formatflowed import convertToWrapped
    from netaddr import IPNetwork
except:
    dopip = True

if dopip and (getpass.getuser() != "root"):
    print("It looks like you need to install some python modules first")
    print("Either run this as root to do so, or run: ")
    print("pip3 install elasticsearch formatflowed netaddr")
    sys.exit(-1)

elif dopip:
    print("Before we get started, we need to install some modules")
    print("Hang on!")
    try:
        subprocess.check_call(('pip3','install','elasticsearch','formatflowed', 'netaddr'))
        from elasticsearch import Elasticsearch
        from formatflowed import convertToWrapped
        from netaddr import IPNetwork
    except:
        print("Oh dear, looks like this failed :(")
        print("Please install elasticsearch and formatflowed before you try again:")
        print("pip install elasticsearch formatflowed netaddr")
        sys.exit(-1)

# CLI arg parsing
parser = argparse.ArgumentParser(description='Command line options.')

parser.add_argument('--defaults', dest='defaults', action='store_true',
                   help='Use default settings')

parser.add_argument('--dbhost', dest='dbhost', type=str, nargs=1,
                   help='ES backend hostname')
parser.add_argument('--dbport', dest='dbport', type=str, nargs=1,
                   help='DB port')
parser.add_argument('--dbname', dest='dbname', type=str, nargs=1,
                   help='ES DB name')
parser.add_argument('--mailserver', dest='mailserver', type=str, nargs=1,
                   help='Host name of outgoing mail server')
parser.add_argument('--mldom', dest='mldom', type=str, nargs=1,
                   help='Domains to accept mail for via UI')
parser.add_argument('--wordcloud', dest='wc', action='store_true',
                   help='Enable word cloud')
parser.add_argument('--skiponexist', dest='soe', action='store_true',
                   help='Skip setup if ES index exists')
parser.add_argument('--noindex', dest='noi', action='store_true',
                   help="Don't make an ES index, assume it exists")
parser.add_argument('--nocloud', dest='nwc', action='store_true',
                   help='Do not enable word cloud')
args = parser.parse_args()

print("Welcome to the Pony Mail setup script!")
print("Let's start by determining some settings...")
print("")

hostname = ""
port = 0
dbname = ""
mlserver = ""
mldom = ""
wc = ""
wce = False

# If called with --defaults (like from Docker), use default values
if args.defaults:
    hostname = "localhost"
    port = 9200
    dbname = "ponymail"
    mlserver = "localhost"
    mldom = "example.org"
    wc = "Y"
    wce = True

# Accept CLI args, copy them
if args.dbhost:
    hostname = args.dbhost[0]
if args.dbport:
    port = int(args.dbport[0])
if args.dbname:
    dbname = args.dbname[0]
if args.mailserver:
    mlserver = args.mailserver[0]
if args.mldom:
    mldom = args.mldom[0]
if args.wc:
    wc = args.wc
if args.nwc:
    wc = False

while hostname == "":
    hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ")

while port < 1:
    try:
        port = int(input("What port is ElasticSearch listening on? (normally 9200): "))
    except ValueError:
        pass

while dbname == "":
    dbname = input("What would you like to call the mail index (e.g. ponymail): ")

while mlserver == "":
    mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ")

while mldom == "":
    mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ")

while wc == "":
    wc = input("Would you like to enable the word cloud feature? (Y/N): ")
    if wc.lower() == "y":
        wce = True

print("Okay, I got all I need, setting up Pony Mail...")

if not args.noi:
    print("Creating index " + dbname)

    es = Elasticsearch([
        {
            'host': hostname,
            'port': port,
            'use_ssl': False,
            'url_prefix': ''
        }],
        max_retries=5,
        retry_on_timeout=True
        )
    mappings = {
        "mbox" : {
          "properties" : {
            "@import_timestamp" : {
              "type" : "date",
              "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
            },
            "@version" : {
              "type" : "long"
            },
            "attachments" : {
              "properties" : {
                "content_type" : {
                  "type" : "string"
                },
                "filename" : {
                  "type" : "string"
                },
                "hash" : {
                  "type" : "string"
                },
                "size" : {
                  "type" : "long"
                }
              }
            },
            "body" : {
              "type" : "string"
            },
            "cc": {
              "type": "string"
            },
            "date" : {
              "type" : "date",
              "store" : True,
              "format" : "yyyy/MM/dd HH:mm:ss",
              "index" : "not_analyzed"
            },
            "epoch" : {
              "type" : "double",
              "index" : "not_analyzed"
            },
            "from" : {
              "type" : "string"
            },
            "from_raw" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "in-reply-to" : {
              "type" : "string"
            },
            "list" : {
              "type" : "string"
            },
            "list_raw" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "message-id" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "mid" : {
              "type" : "string"
            },
            "private" : {
              "type" : "boolean"
            },
            "references" : {
              "type" : "string"
            },
            "subject" : {
              "type" : "string"
            },
            "to" : {
              "type" : "string"
            }
          }
        },
        "attachment" : {
          "properties" : {
            "source" : {
              "type" : "binary"
            }
          }
        },
        "mbox_source" : {
          "properties" : {
            "source" : {
              "type" : "binary"
            },
            "@version" : {
              "type" : "long"
            },
            "message-id" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "mid" : {
              "type" : "string"
            }
          }
        },
        "mailinglists" : {
          "properties" : {
            "description" : {
              "type" : "string"
            },
            "list" : {
              "type" : "string"
            },
            "name" : {
              "type" : "string"
            }
          }
        },
        "account" : {
          "properties" : {
            "cid" : {
              "type" : "string"
            },
            "credentials" : {
              "properties" : {
                "altemail" : {
                  "type" : "object"
                },
                "email" : {
                  "type" : "string"
                },
                "fullname" : {
                  "type" : "string"
                },
                "uid" : {
                  "type" : "string"
                }
              }
            },
            "internal" : {
              "properties" : {
                "cookie" : {
                  "type" : "string"
                },
                "ip" : {
                  "type" : "string"
                },
                "oauth_used" : {
                  "type" : "string"
                }
              }
            },
            "request_id" : {
              "type" : "string"
            }
          }
        },
        "notifications" : {
          "properties" : {
            "date" : {
              "type" : "date",
              "store" : True,
              "format" : "yyyy/MM/dd HH:mm:ss"
            },
            "epoch" : {
              "type" : "double"
            },
            "from" : {
              "type" : "string"
            },
            "in-reply-to" : {
              "type" : "string"
            },
            "list" : {
              "type" : "string"
            },
            "message-id" : {
              "type" : "string",
              "index" : "not_analyzed"
            },
            "mid" : {
              "type" : "string"
            },
            "private" : {
              "type" : "boolean"
            },
            "recipient" : {
              "type" : "string"
            },
            "seen" : {
              "type" : "long"
            },
            "subject" : {
              "type" : "string"
            },
            "to" : {
              "type" : "string"
            },
            "type" : {
              "type" : "string"
            }
          }
        }
    }

    # Check if index already exists
    if es.indices.exists(dbname):
        if args.soe:
            print("Index already exists and SOE set, exiting quietly")
            sys.exit(0)
        else:
            print("Error: ElasticSearch index %s already exists!" % dbname)
            sys.exit(-1)

    res = es.indices.create(index = dbname, body = {
                "mappings" : mappings
            }
        )

    print("Index created!")

print("Writing importer config (ponymail.cfg)")

with open("ponymail.cfg", "w") as f:
    f.write("""
###############################################################
# Pony Mail Configuration file

# Main ES configuration
[elasticsearch]
hostname:               %s
dbname:                 %s
port:                   %u
ssl:                    false

#uri:                   url_prefix

#user:                  username
#password:              password

#write:                 consistency level (default quorum)

#backup:                database name

[archiver]
#generator:             medium|full|other

[debug]
#cropout:               string to crop from list-id

###############################################################
            """ % (hostname, dbname, port))
    f.close()

print("mod_lua configuration (config.lua)")
with open("../site/api/lib/config.lua", "w") as f:
    f.write("""
local config = {
    es_url = "http://%s:%u/%s/",
    mailserver = "%s",
    accepted_domains = "%s",
    wordcloud = %s,
    slow_count = false,
    email_footer = nil, -- see the docs for how to set this up.
    full_headers = false,
    maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists
    admin_oauth = {}, -- list of domains that may do administrative oauth (private list access)
                     -- add 'www.googleapis.com' to the list for google oauth to decide, for instance.
    oauth_fields = {}, -- used for specifying individual oauth handling parameters.
    antispam = true  -- Whether or not to add anti-spam measures aimed at anonymous users.
    -- no_association = {} -- domains that are not allowed for email association
    -- hidePrivate = true -- whether to hide names of lists that contain any private mails
    -- listsDisplay = 'regex' -- if defined, hide list names that don't match the regex
}
return config
            """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
    f.close()

print("Copying sample JS config to config.js (if needed)...")
if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"):
    shutil.copy("../site/js/config.js.sample", "../site/js/config.js")

print("All done, Pony Mail should...work now :)")
print("If you are using an external mail inbound server, \nmake sure to copy archiver.py and ponymail.cfg to it")
<|endoftext|>