<commit_before>#     Copyright 2014, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Utility module.

Here the small things for file/dir names, Python version, CPU counting,
memory usage, etc. that fit nowhere else and don't deserve their own names.

"""

import sys, os, subprocess

def _getPythonVersion():
    big, major, minor = sys.version_info[0:3]

    return big * 100 + major * 10 + minor

python_version = _getPythonVersion()

def getOS():
    if os.name == "nt":
        return "Windows"
    elif os.name == "posix":
        return os.uname()[0]
    else:
        assert False, os.name

def getArchitecture():
    if getOS() == "Windows":
        if "AMD64" in sys.version:
            return "x86_64"
        else:
            return "x86"
    else:
        return os.uname()[4]

def relpath(path):
    return os.path.relpath( path )

def abspath(path):
    return os.path.abspath( path )

def joinpath(*parts):
    return os.path.join( *parts )

def splitpath(path):
    return tuple( element for element in os.path.split( path ) if element )

def basename(path):
    return os.path.basename( path )

def dirname(path):
    return os.path.dirname( path )

def normpath(path):
    return os.path.normpath( path )

def normcase(path):
    return os.path.normcase( path )

def getExtension(path):
    return os.path.splitext( path )[1]

def isFile(path):
    return os.path.isfile( path )

def isDir(path):
    return os.path.isdir( path )

def listDir(path):
    """ Give a sorted path, basename pairs of a directory."""

    return sorted(
        [
            ( joinpath( path, filename ), filename )
            for filename in
            os.listdir( path )
        ]
    )

def deleteFile(path, must_exist):
    if must_exist or isFile( path ):
        os.unlink( path )

def makePath(path):
    os.makedirs( path )

def getCoreCount():
    cpu_count = 0

    # Try to sum up the CPU cores, if the kernel shows them.
    try:
        # Try to get the number of logical processors
        with open( "/proc/cpuinfo" ) as cpuinfo_file:
            cpu_count = cpuinfo_file.read().count( "processor\t:" )
    except IOError:
        pass

    if not cpu_count:
        # false alarm, no re-import, just a function level import to avoid it
        # unless it is absolutely necessary, pylint: disable=W0404

        import multiprocessing
        cpu_count = multiprocessing.cpu_count()

    return cpu_count

def callExec(args):
    """ Do exec in a portable way preserving exit code.

        On Windows, unfortunately there is no real exec, so we have to spawn
        a new process instead.
    """

    # On Windows os.execl does not work properly
    if getOS() != "Windows":
        # The star arguments is the API of execl, pylint: disable=W0142
        os.execl(*args)
    else:
        args = list(args)
        del args[1]
        sys.exit(
            subprocess.call(args)
        )

def encodeNonAscii(var_name):
    """ Encode variable name that is potentially not ASCII to ASCII only.

        For Python3, unicode identifiers can be used, but these are not
        possible in C++03, so we need to replace them.
    """
    if python_version < 300:
        return var_name
    else:
        var_name = var_name.encode( "ascii", "xmlcharrefreplace" )
        var_name = var_name.decode( "ascii" )

        # TODO: Is this truly safe of collisions, I think it is not. It might be
        # necessary to use something that is not allowed otherwise.
        return var_name.replace( "&#", "$$" ).replace( ";", "" )

def getOwnProcessMemoryUsage():
    """ Memory usage of own process in bytes.

    """

    if os.name == "nt":

        # adapted from http://code.activestate.com/recipes/578513
        import ctypes
        from ctypes import wintypes

        # Lets allow this to match Windows API it reflects,
        # pylint: disable=C0103
        class PROCESS_MEMORY_COUNTERS_EX(ctypes.Structure):
            _fields_ = [
                ('cb', wintypes.DWORD),
                ('PageFaultCount', wintypes.DWORD),
                ('PeakWorkingSetSize', ctypes.c_size_t),
                ('WorkingSetSize', ctypes.c_size_t),
                ('QuotaPeakPagedPoolUsage', ctypes.c_size_t),
                ('QuotaPagedPoolUsage', ctypes.c_size_t),
                ('QuotaPeakNonPagedPoolUsage', ctypes.c_size_t),
                ('QuotaNonPagedPoolUsage', ctypes.c_size_t),
                ('PagefileUsage', ctypes.c_size_t),
                ('PeakPagefileUsage', ctypes.c_size_t),
                ('PrivateUsage', ctypes.c_size_t),
            ]

        GetProcessMemoryInfo = ctypes.windll.psapi.GetProcessMemoryInfo
        GetProcessMemoryInfo.argtypes = [
            wintypes.HANDLE,
            ctypes.POINTER(PROCESS_MEMORY_COUNTERS_EX),
            wintypes.DWORD,
        ]
        GetProcessMemoryInfo.restype = wintypes.BOOL

        counters = PROCESS_MEMORY_COUNTERS_EX()
        rv = GetProcessMemoryInfo(
            ctypes.windll.kernel32.GetCurrentProcess(),
            ctypes.byref(counters),
            ctypes.sizeof(counters)
        )

        if not rv:
            raise ctypes.WinError()

        return counters.PrivateUsage
    else:
        import resource

        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024

def getHumanReadableProcessMemoryUsage(value = None):
    if value is None:
        value = getOwnProcessMemoryUsage()

    if abs(value) < 1024*1014:
        return "%.2f KB (%d bytes)" % (
            value / 1024.0,
            value
        )
    elif abs(value) < 1024*1014*1024:
        return "%.2f MB (%d bytes)" % (
            value / (1024*1024.0),
            value
        )
    elif abs(value) < 1024*1014*1024*1024:
        return "%.2f GB (%d bytes)" % (
            value / (1024*1024*1024.0),
            value
        )
    else:
        return "%d bytes" % value

class MemoryWatch:
    def __init__(self):
        self.start = getOwnProcessMemoryUsage()
        self.stop = None

    def finish(self):
        self.stop = getOwnProcessMemoryUsage()

    def asStr(self):
        return getHumanReadableProcessMemoryUsage(self.stop - self.start)
<commit_msg>Windows: Handle cases, where filenames are on other volumnes.<commit_after>#     Copyright 2014, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Utility module.

Here the small things for file/dir names, Python version, CPU counting,
memory usage, etc. that fit nowhere else and don't deserve their own names.

"""

import sys, os, subprocess

def _getPythonVersion():
    big, major, minor = sys.version_info[0:3]

    return big * 100 + major * 10 + minor

python_version = _getPythonVersion()

def getOS():
    if os.name == "nt":
        return "Windows"
    elif os.name == "posix":
        return os.uname()[0]
    else:
        assert False, os.name

def getArchitecture():
    if getOS() == "Windows":
        if "AMD64" in sys.version:
            return "x86_64"
        else:
            return "x86"
    else:
        return os.uname()[4]

def relpath(path):
    try:
        return os.path.relpath(path)
    except ValueError:
        # On Windows, paths on different devices prevent it to work. Use that
        # full path then.
        if getOS() == "Windows":
            return os.path.abspath(path)
        raise

def abspath(path):
    return os.path.abspath(path)

def joinpath(*parts):
    return os.path.join( *parts )

def splitpath(path):
    return tuple(
        element
        for element in
        os.path.split(path)
        if element
    )

def basename(path):
    return os.path.basename(path)

def dirname(path):
    return os.path.dirname(path)

def normpath(path):
    return os.path.normpath(path)

def normcase(path):
    return os.path.normcase(path)

def getExtension(path):
    return os.path.splitext(path)[1]

def isFile(path):
    return os.path.isfile(path)

def isDir(path):
    return os.path.isdir(path)

def listDir(path):
    """ Give a sorted path, basename pairs of a directory."""

    return sorted(
        [
            (
                joinpath(path, filename),
                filename
            )
            for filename in
            os.listdir(path)
        ]
    )

def deleteFile(path, must_exist):
    if must_exist or isFile( path ):
        os.unlink( path )

def makePath(path):
    os.makedirs( path )

def getCoreCount():
    cpu_count = 0

    # Try to sum up the CPU cores, if the kernel shows them.
    try:
        # Try to get the number of logical processors
        with open( "/proc/cpuinfo" ) as cpuinfo_file:
            cpu_count = cpuinfo_file.read().count( "processor\t:" )
    except IOError:
        pass

    if not cpu_count:
        # false alarm, no re-import, just a function level import to avoid it
        # unless it is absolutely necessary, pylint: disable=W0404

        import multiprocessing
        cpu_count = multiprocessing.cpu_count()

    return cpu_count

def callExec(args):
    """ Do exec in a portable way preserving exit code.

        On Windows, unfortunately there is no real exec, so we have to spawn
        a new process instead.
    """

    # On Windows os.execl does not work properly
    if getOS() != "Windows":
        # The star arguments is the API of execl, pylint: disable=W0142
        os.execl(*args)
    else:
        args = list(args)
        del args[1]
        sys.exit(
            subprocess.call(args)
        )

def encodeNonAscii(var_name):
    """ Encode variable name that is potentially not ASCII to ASCII only.

        For Python3, unicode identifiers can be used, but these are not
        possible in C++03, so we need to replace them.
    """
    if python_version < 300:
        return var_name
    else:
        var_name = var_name.encode("ascii", "xmlcharrefreplace")
        var_name = var_name.decode("ascii")

        # TODO: Is this truly safe of collisions, I think it is not. It might be
        # necessary to use something that is not allowed otherwise.
        return var_name.replace("&#", "$$").replace(";", "")

def getOwnProcessMemoryUsage():
    """ Memory usage of own process in bytes.

    """

    if os.name == "nt":
        # adapted from http://code.activestate.com/recipes/578513
        import ctypes
        from ctypes import wintypes

        # Lets allow this to match Windows API it reflects,
        # pylint: disable=C0103
        class PROCESS_MEMORY_COUNTERS_EX(ctypes.Structure):
            _fields_ = [
                ('cb', wintypes.DWORD),
                ('PageFaultCount', wintypes.DWORD),
                ('PeakWorkingSetSize', ctypes.c_size_t),
                ('WorkingSetSize', ctypes.c_size_t),
                ('QuotaPeakPagedPoolUsage', ctypes.c_size_t),
                ('QuotaPagedPoolUsage', ctypes.c_size_t),
                ('QuotaPeakNonPagedPoolUsage', ctypes.c_size_t),
                ('QuotaNonPagedPoolUsage', ctypes.c_size_t),
                ('PagefileUsage', ctypes.c_size_t),
                ('PeakPagefileUsage', ctypes.c_size_t),
                ('PrivateUsage', ctypes.c_size_t),
            ]

        GetProcessMemoryInfo = ctypes.windll.psapi.GetProcessMemoryInfo
        GetProcessMemoryInfo.argtypes = [
            wintypes.HANDLE,
            ctypes.POINTER(PROCESS_MEMORY_COUNTERS_EX),
            wintypes.DWORD,
        ]
        GetProcessMemoryInfo.restype = wintypes.BOOL

        counters = PROCESS_MEMORY_COUNTERS_EX()
        rv = GetProcessMemoryInfo(
            ctypes.windll.kernel32.GetCurrentProcess(),
            ctypes.byref(counters),
            ctypes.sizeof(counters)
        )

        if not rv:
            raise ctypes.WinError()

        return counters.PrivateUsage
    else:
        import resource

        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024

def getHumanReadableProcessMemoryUsage(value = None):
    if value is None:
        value = getOwnProcessMemoryUsage()

    if abs(value) < 1024*1014:
        return "%.2f KB (%d bytes)" % (
            value / 1024.0,
            value
        )
    elif abs(value) < 1024*1014*1024:
        return "%.2f MB (%d bytes)" % (
            value / (1024*1024.0),
            value
        )
    elif abs(value) < 1024*1014*1024*1024:
        return "%.2f GB (%d bytes)" % (
            value / (1024*1024*1024.0),
            value
        )
    else:
        return "%d bytes" % value

class MemoryWatch:
    def __init__(self):
        self.start = getOwnProcessMemoryUsage()
        self.stop = None

    def finish(self):
        self.stop = getOwnProcessMemoryUsage()

    def asStr(self):
        return getHumanReadableProcessMemoryUsage(self.stop - self.start)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import serial
import re

parser = argparse.ArgumentParser(
  description='Load ADIF from a QRPworks HCT or SideKar module.',
  epilog="use 'python -m serial.tools.list_ports' to find port")
parser.add_argument('port',help='serial port to be used')
parser.add_argument('output',
  help='.adi file to be written (WILL BE OVERWRITTEN)')
args = parser.parse_args()

ser = serial.Serial(args.port,timeout=2)

outfile = open(args.output,'w')

adif_rem = -1

print 'Waiting for ADIF...'

while True:
  hct_in = ser.read(512)
  print hct_in
  if '{' in hct_in and adif_rem == -1:
    adif_rem = int(re.split('[{}]',hct_in)[1])
  elif adif_rem > 0:
    if '[' in hct_in:
      hct_in = hct_in.split('[')[1]
    if len(hct_in) >= adif_rem:
      outfile.write(hct_in[:adif_rem])
      break
    else:
      outfile.write(hct_in)
      adif_rem = adif_rem - len(hct_in)
  if not adif_rem:
    break
<commit_msg>Modified to look for end, documented<commit_after>#!/usr/bin/env python

import argparse
import serial
import re

# This should hopefully be self-explanatory
parser = argparse.ArgumentParser(
  description='Load ADIF from a QRPworks HCT or SideKar module.',
  epilog="use 'python -m serial.tools.list_ports' to find port")
parser.add_argument('port',help='serial port to be used')
parser.add_argument('output',
  help='.adi file to be written (WILL BE OVERWRITTEN)')
args = parser.parse_args()

# Open serial port. Errors passed to user. Timeout may be excessive.
ser = serial.Serial(args.port,timeout=2)

# Open output file. Errors, again, passed to user.
outfile = open(args.output,'w')

# Prep this variable. We assume we don't start in ADIF.
in_adif = False

print 'Waiting for ADIF...'

# This is an unbounded, but not infinite, loop.
# The code within should exit.
while True:
  hct_in = ser.read(512) # Read up to 512 bytes.
  # If we're not in ADIF, find the start
  if not in_adif: # Data not started yet
    if '[' in hct_in: # if ADIF starts here:
      in_adif = True # set in_adif for future loops
      hct_in = hct_in.split('[',1)[1] # strip any preceeding data
    else: # otherwise,
      continue # jump to the next loop - more data!
  # The continue line above should guarantee this
  assert in_adif, "BUG: in_adif not set, but ADIF input reached"
  # Look for the end of the ADIF data
  if '\n]' in hct_in: # If the end of ADIF happens in collected data:
    in_adif = False # Done after this loop
    hct_in = hct_in.rsplit(']',1)[0] # Strip late data
  # The next statement deals with an edge case:
  # what happens if '\n]' happens across reads?
  if hct_in.ends_with('\n'): # If the read buffer ends with newline
    next_byte = ser.read(1) # Read one more byte...
    if next_byte == ']': # if it's a close brace
      in_adif = False # we're done with ADIF; throw it away
    else: # otherwise
      hct_in = hct_in + next_byte # append the next byte to hct_in
  # We have our current ADIF data. Dump it.
  outfile.write(hct_in) # write the current data to the file
  # Check if we should loop again
  if not in_adif: # If we're done with ADIF
    break # Exit the while loop, terminate the script.
<|endoftext|>


<commit_before>'''
Created on Mar 6, 2013

@author: Gary

'''
import os
import struct
from datetime import datetime, timedelta
from xbee import ZigBee
from lib.common import Common
from lib.constants import Constants
from lib.base import Base
from inputs.zigbeeinput.beaglebonexbeecommunications import BeagleboneXbeeCommunications
from inputs.zigbeeinput.windowsxbeecommunications import WindowsXbeeCommunications

class UnsupportedSystemError( Exception ):
    pass

class ZigBeeOutput( Base, object ):
    '''
    ZigBeeOutput is responsible for formating a ZigBee message and sending it to a remote XBee.
    '''
    setPinHigh = b'\x05'
    setPinLow = b'\x04'
    serial = None
    zigbee = None
    previous_datetime = datetime.utcnow()

    deviceToCommand = {'DIO-0': b'D0',
                    'DIO-1': b'D1',
                    'DIO-2': b'D2',
                    'DIO-3': b'D3',
                    'DIO-4': b'D4',
                    'DIO-5': b'D5',
                    'DIO-6': b'D6',
                    'DIO-7': b'D7'}

    selectHighOrLow = {False: setPinLow, True: setPinHigh}
    communication_module = {'posix': BeagleboneXbeeCommunications,
                            'nt': WindowsXbeeCommunications}

    def __init__( self ):
        '''
        '''
        super( ZigBeeOutput, self ).__init__()

    def startCorrectZigbee( self, os_name=os.name ):
        if ( os_name in self.communication_module ):
            self.comm = self.communication_module[os_name]()
            self.serial = self.comm.setup()
            self.zigbee = ZigBee( self.serial )
        else:
            raise UnsupportedSystemError( "System {} not supported".format( os_name ) )

    @property
    def logger_name( self ):
        ''' Set the logger level. '''
        return Constants.LogKeys.outputsZigBee

    def sendCommand( self, value, data ):

        try:
            device, port = Common.getDeviceAndPort( data )
            dest_addr_long = struct.pack( '!Q', int( device, 16 ) )
            command = self.deviceToCommand[port.upper()]
            parameter = self.selectHighOrLow[value]
            current_datetime = datetime.utcnow()
            delta = current_datetime - self.previous_datetime
            self.logger.info( '{} dest_addr = {:x} command = {} parameter {:x}'.format( str( delta ).split( '.' )[0], struct.unpack( '!Q', dest_addr_long )[0], command, struct.unpack( 'B', parameter )[0] ) )
            self.zigbee.remote_at( dest_addr_long=dest_addr_long, frame_id=b'\xaa', command=command, parameter=parameter )
            self.previous_datetime = current_datetime
        except KeyError:
            self.logger.exception( 'KeyError exception: value = {} data = {}'.format( value, data ) )
<commit_msg>Incomplete - task : Document HouseMonitor  Clean up<commit_after>'''
Created on Mar 6, 2013

@author: Gary

'''
import os
import struct
from datetime import datetime, timedelta
from xbee import ZigBee
from lib.common import Common
from lib.constants import Constants
from lib.base import Base
from inputs.zigbeeinput.beaglebonexbeecommunications import BeagleboneXbeeCommunications
from inputs.zigbeeinput.windowsxbeecommunications import WindowsXbeeCommunications

class UnsupportedSystemError( Exception ):
    pass

class ZigBeeOutput( Base, object ):
    '''
    ZigBeeOutput is responsible for formating a ZigBee message and sending it to a remote XBee.
    '''
    setPinHigh = b'\x05'
    setPinLow = b'\x04'
    serial = None
    zigbee = None
    previous_datetime = datetime.utcnow()

    deviceToCommand = {'DIO-0': b'D0',
                    'DIO-1': b'D1',
                    'DIO-2': b'D2',
                    'DIO-3': b'D3',
                    'DIO-4': b'D4',
                    'DIO-5': b'D5',
                    'DIO-6': b'D6',
                    'DIO-7': b'D7'}

    selectHighOrLow = {False: setPinLow, True: setPinHigh}
    communication_module = {'posix': BeagleboneXbeeCommunications,
                            'nt': WindowsXbeeCommunications}

    def __init__( self ):
        '''
        '''
        super( ZigBeeOutput, self ).__init__()

    def startCorrectZigbee( self, os_name=os.name ):
        if ( os_name in self.communication_module ):
            self.comm = self.communication_module[os_name]()
            self.serial = self.comm.setup()
            self.zigbee = ZigBee( self.serial )
        else:
            raise UnsupportedSystemError( "System {} not supported".format( os_name ) )

    @property
    def logger_name( self ):
        ''' Set the logger level. '''
        return Constants.LogKeys.outputsZigBee

    def sendCommand( self, value, data ):

        try:
            device, port = Common.getDeviceAndPort( data )
            dest_addr_long = struct.pack( '!Q', int( device, 16 ) )
            command = self.deviceToCommand[port.upper()]
            parameter = self.selectHighOrLow[value]
            current_datetime = datetime.utcnow()
            delta = current_datetime - self.previous_datetime
            self.logger.info( '{} dest_addr = {:x} command = {} parameter {:x}'.
                              format( str( delta ).split( '.' )[0],
                                      struct.unpack( '!Q', dest_addr_long )[0], command,
                                      struct.unpack( 'B', parameter )[0] ) )
            self.zigbee.remote_at( dest_addr_long=dest_addr_long, frame_id=b'\xaa',
                                   command=command,
                                   parameter=parameter )
            self.previous_datetime = current_datetime
        except KeyError:
            self.logger.exception( 'KeyError exception: value = {} data = {}'.format( value, data ) )
<|endoftext|>


<commit_before>##
# Copyright (c) 2010 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
File addressbook store tests.
"""

from twext.python.filepath import CachingFilePath as FilePath
from twisted.trial import unittest

from twistedcaldav.vcard import Component as VComponent

from txdav.common.icommondatastore import HomeChildNameNotAllowedError
from txdav.common.icommondatastore import ObjectResourceNameNotAllowedError
from txdav.common.icommondatastore import ObjectResourceUIDAlreadyExistsError
from txdav.common.icommondatastore import NoSuchHomeChildError
from txdav.common.icommondatastore import NoSuchObjectResourceError

from txdav.carddav.datastore.file import AddressBookStore, AddressBookHome
from txdav.carddav.datastore.file import AddressBook, AddressBookObject

from txdav.carddav.datastore.test.common import (
    CommonTests, vcard4_text, vcard1modified_text, StubNotifierFactory)

storePath = FilePath(__file__).parent().child("addressbook_store")

def _todo(f, why):
    f.todo = why
    return f

featureUnimplemented = lambda f: _todo(f, "Feature unimplemented")
testUnimplemented = lambda f: _todo(f, "Test unimplemented")
todo = lambda why: lambda f: _todo(f, why)

def setUpAddressBookStore(test):
    test.root = FilePath(test.mktemp())
    test.root.createDirectory()

    storeRootPath = test.storeRootPath = test.root.child("store")
    addressbookPath = storeRootPath.child("addressbooks").child("__uids__")
    addressbookPath.parent().makedirs()
    storePath.copyTo(addressbookPath)

    test.notifierFactory = StubNotifierFactory()
    test.addressbookStore = AddressBookStore(storeRootPath, test.notifierFactory)
    test.txn = test.addressbookStore.newTransaction()
    assert test.addressbookStore is not None, "No addressbook store?"

def setUpHome1(test):
    setUpAddressBookStore(test)
    test.home1 = test.txn.addressbookHomeWithUID("home1")
    assert test.home1 is not None, "No addressbook home?"

def setUpAddressBook1(test):
    setUpHome1(test)
    test.addressbook1 = test.home1.addressbookWithName("addressbook_1")
    assert test.addressbook1 is not None, "No addressbook?"

class AddressBookStoreTest(unittest.TestCase):
    """
    Test cases for L{AddressBookStore}.
    """

    def setUp(self):
        setUpAddressBookStore(self)

    def test_addressbookHomeWithUID_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no UIDs may start with ".".
        """
        self.assertEquals(
            self.addressbookStore.newTransaction().addressbookHomeWithUID("xyzzy"),
            None
        )

class AddressBookHomeTest(unittest.TestCase):

    def setUp(self):
        setUpHome1(self)

    def test_init(self):
        """
        L{AddressBookHome} has C{_path} and L{_addressbookStore} attributes,
        indicating its location on disk and parent store, respectively.
        """
        self.failUnless(
            isinstance(self.home1._path, FilePath),
            self.home1._path
        )
        self.assertEquals(
            self.home1._addressbookStore,
            self.addressbookStore
        )

    def test_addressbookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        name = ".foo"
        self.home1._path.child(name).createDirectory()
        self.assertEquals(self.home1.addressbookWithName(name), None)

    def test_createAddressBookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        self.assertRaises(
            HomeChildNameNotAllowedError,
            self.home1.createAddressBookWithName, ".foo"
        )

    def test_removeAddressBookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        name = ".foo"
        self.home1._path.child(name).createDirectory()
        self.assertRaises(
            NoSuchHomeChildError,
            self.home1.removeAddressBookWithName, name
        )

class AddressBookTest(unittest.TestCase):

    def setUp(self):
        setUpAddressBook1(self)

    def test_init(self):
        """
        L{AddressBook.__init__} sets private attributes to reflect its constructor
        arguments.
        """
        self.failUnless(
            isinstance(self.addressbook1._path, FilePath),
            self.addressbook1
        )
        self.failUnless(
            isinstance(self.addressbook1._addressbookHome, AddressBookHome),
            self.addressbook1._addressbookHome
        )

    def test_useIndexImmediately(self):
        """
        L{AddressBook._index} is usable in the same transaction it is created, with
        a temporary filename.
        """
        self.home1.createAddressBookWithName("addressbook2")
        addressbook = self.home1.addressbookWithName("addressbook2")
        index = addressbook._index
        self.assertEquals(set(index.addressbookObjects()),
                          set(addressbook.addressbookObjects()))
        self.txn.commit()
        self.txn = self.addressbookStore.newTransaction()
        self.home1 = self.txn.addressbookHomeWithUID("home1")
        addressbook = self.home1.addressbookWithName("addressbook2")
        # FIXME: we should be curating our own index here, but in order to fix
        # that the code in the old implicit scheduler needs to change.  This
        # test would be more effective if there were actually some objects in
        # this list.
        index = addressbook._index
        self.assertEquals(set(index.addressbookObjects()),
                          set(addressbook.addressbookObjects()))

    def test_addressbookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        name = ".foo.vcf"
        self.home1._path.child(name).touch()
        self.assertEquals(self.addressbook1.addressbookObjectWithName(name), None)

    @featureUnimplemented
    def test_addressbookObjectWithUID_exists(self):
        """
        Find existing addressbook object by name.
        """
        addressbookObject = self.addressbook1.addressbookObjectWithUID("1")
        self.failUnless(
            isinstance(addressbookObject, AddressBookObject),
            addressbookObject
        )
        self.assertEquals(
            addressbookObject.component(),
            self.addressbook1.addressbookObjectWithName("1.vcf").component()
        )

    def test_createAddressBookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        self.assertRaises(
            ObjectResourceNameNotAllowedError,
            self.addressbook1.createAddressBookObjectWithName,
            ".foo", VComponent.fromString(vcard4_text)
        )

    @featureUnimplemented
    def test_createAddressBookObjectWithName_uidconflict(self):
        """
        Attempt to create a addressbook object with a conflicting UID
        should raise.
        """
        name = "foo.vcf"
        assert self.addressbook1.addressbookObjectWithName(name) is None
        component = VComponent.fromString(vcard1modified_text)
        self.assertRaises(
            ObjectResourceUIDAlreadyExistsError,
            self.addressbook1.createAddressBookObjectWithName,
            name, component
        )

    def test_removeAddressBookObject_delayedEffect(self):
        """
        Removing a addressbook object should not immediately remove the underlying
        file; it should only be removed upon commit() of the transaction.
        """
        self.addressbook1.removeAddressBookObjectWithName("2.vcf")
        self.failUnless(self.addressbook1._path.child("2.vcf").exists())
        self.txn.commit()
        self.failIf(self.addressbook1._path.child("2.vcf").exists())

    def test_removeAddressBookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        name = ".foo"
        self.addressbook1._path.child(name).touch()
        self.assertRaises(
            NoSuchObjectResourceError,
            self.addressbook1.removeAddressBookObjectWithName, name
        )

    def _refresh(self):
        """
        Re-read the (committed) home1 and addressbook1 objects in a new
        transaction.
        """
        self.txn = self.addressbookStore.newTransaction()
        self.home1 = self.txn.addressbookHomeWithUID("home1")
        self.addressbook1 = self.home1.addressbookWithName("addressbook_1")

    def test_undoCreateAddressBookObject(self):
        """
        If a addressbook object is created as part of a transaction, it will be
        removed if that transaction has to be aborted.
        """
        # Make sure that the addressbook home is actually committed; rolling back
        # addressbook home creation will remove the whole directory.
        self.txn.commit()
        self._refresh()
        self.addressbook1.createAddressBookObjectWithName(
            "sample.vcf",
            VComponent.fromString(vcard4_text)
        )
        self._refresh()
        self.assertIdentical(
            self.addressbook1.addressbookObjectWithName("sample.vcf"),
            None
        )

    def doThenUndo(self):
        """
        Commit the current transaction, but add an operation that will cause it
        to fail at the end.  Finally, refresh all attributes with a new
        transaction so that further oparations can be performed in a valid
        context.
        """
        def fail():
            raise RuntimeError("oops")
        self.txn.addOperation(fail, "dummy failing operation")
        self.assertRaises(RuntimeError, self.txn.commit)
        self._refresh()

    def test_undoModifyAddressBookObject(self):
        """
        If an existing addressbook object is modified as part of a transaction, it
        should be restored to its previous status if the transaction aborts.
        """
        originalComponent = self.addressbook1.addressbookObjectWithName(
            "1.vcf").component()
        self.addressbook1.addressbookObjectWithName("1.vcf").setComponent(
            VComponent.fromString(vcard1modified_text)
        )
        # Sanity check.
        self.assertEquals(
            self.addressbook1.addressbookObjectWithName("1.vcf").component(),
            VComponent.fromString(vcard1modified_text)
        )
        self.doThenUndo()
        self.assertEquals(
            self.addressbook1.addressbookObjectWithName("1.vcf").component(),
            originalComponent
        )

    def test_modifyAddressBookObjectCaches(self):
        """
        Modifying a addressbook object should cache the modified component in
        memory, to avoid unnecessary parsing round-trips.
        """
        modifiedComponent = VComponent.fromString(vcard1modified_text)
        self.addressbook1.addressbookObjectWithName("1.vcf").setComponent(
            modifiedComponent
        )
        self.assertIdentical(
            modifiedComponent,
            self.addressbook1.addressbookObjectWithName("1.vcf").component()
        )

    @featureUnimplemented
    def test_removeAddressBookObjectWithUID_absent(self):
        """
        Attempt to remove an non-existing addressbook object should raise.
        """
        self.assertRaises(
            NoSuchObjectResourceError,
            self.addressbook1.removeAddressBookObjectWithUID, "xyzzy"
        )

    @testUnimplemented
    def test_syncToken(self):
        """
        Sync token is correct.
        """
        raise NotImplementedError()

    @testUnimplemented
    def test_addressbookObjectsInTimeRange(self):
        """
        Find addressbook objects occuring in a given time range.
        """
        raise NotImplementedError()

    @testUnimplemented
    def test_addressbookObjectsSinceToken(self):
        """
        Find addressbook objects that have been modified since a given
        sync token.
        """
        raise NotImplementedError()

class AddressBookObjectTest(unittest.TestCase):
    def setUp(self):
        setUpAddressBook1(self)
        self.object1 = self.addressbook1.addressbookObjectWithName("1.vcf")

    def test_init(self):
        """
        L{AddressBookObject} has instance attributes, C{_path} and C{_addressbook},
        which refer to its position in the filesystem and the addressbook in which
        it is contained, respectively.
        """
        self.failUnless(
            isinstance(self.object1._path, FilePath),
            self.object1._path
        )
        self.failUnless(
            isinstance(self.object1._addressbook, AddressBook),
            self.object1._addressbook
        )

class FileStorageTests(unittest.TestCase, CommonTests):
    """
    File storage tests.
    """

    def storeUnderTest(self):
        """
        Create and return a L{AddressBookStore} for testing.
        """
        setUpAddressBookStore(self)
        return self.addressbookStore

    def test_init(self):
        """
        L{AddressBookStore} has a C{_path} attribute which refers to its
        constructor argument.
        """
        self.assertEquals(self.storeUnderTest()._path,
                          self.storeRootPath)

    def test_addressbookObjectsWithDotFile(self):
        """
        Adding a dotfile to the addressbook home should not increase
        """
        self.homeUnderTest()._path.child(".foo").createDirectory()
        self.test_addressbookObjects()

<commit_msg>save addressbook store, as calendar file tests do (fix tests)<commit_after>##
# Copyright (c) 2010 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
File addressbook store tests.
"""

from twext.python.filepath import CachingFilePath as FilePath
from twisted.trial import unittest

from twistedcaldav.vcard import Component as VComponent

from txdav.common.icommondatastore import HomeChildNameNotAllowedError
from txdav.common.icommondatastore import ObjectResourceNameNotAllowedError
from txdav.common.icommondatastore import ObjectResourceUIDAlreadyExistsError
from txdav.common.icommondatastore import NoSuchHomeChildError
from txdav.common.icommondatastore import NoSuchObjectResourceError

from txdav.carddav.datastore.file import AddressBookStore, AddressBookHome
from txdav.carddav.datastore.file import AddressBook, AddressBookObject

from txdav.carddav.datastore.test.common import (
    CommonTests, vcard4_text, vcard1modified_text, StubNotifierFactory)

storePath = FilePath(__file__).parent().child("addressbook_store")

def _todo(f, why):
    f.todo = why
    return f

featureUnimplemented = lambda f: _todo(f, "Feature unimplemented")
testUnimplemented = lambda f: _todo(f, "Test unimplemented")
todo = lambda why: lambda f: _todo(f, why)

def setUpAddressBookStore(test):
    test.root = FilePath(test.mktemp())
    test.root.createDirectory()

    storeRootPath = test.storeRootPath = test.root.child("store")
    addressbookPath = storeRootPath.child("addressbooks").child("__uids__")
    addressbookPath.parent().makedirs()
    storePath.copyTo(addressbookPath)

    test.notifierFactory = StubNotifierFactory()
    test.addressbookStore = AddressBookStore(storeRootPath, test.notifierFactory)
    test.txn = test.addressbookStore.newTransaction()
    assert test.addressbookStore is not None, "No addressbook store?"

def setUpHome1(test):
    setUpAddressBookStore(test)
    test.home1 = test.txn.addressbookHomeWithUID("home1")
    assert test.home1 is not None, "No addressbook home?"

def setUpAddressBook1(test):
    setUpHome1(test)
    test.addressbook1 = test.home1.addressbookWithName("addressbook_1")
    assert test.addressbook1 is not None, "No addressbook?"

class AddressBookStoreTest(unittest.TestCase):
    """
    Test cases for L{AddressBookStore}.
    """

    def setUp(self):
        setUpAddressBookStore(self)

    def test_addressbookHomeWithUID_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no UIDs may start with ".".
        """
        self.assertEquals(
            self.addressbookStore.newTransaction().addressbookHomeWithUID("xyzzy"),
            None
        )

class AddressBookHomeTest(unittest.TestCase):

    def setUp(self):
        setUpHome1(self)

    def test_init(self):
        """
        L{AddressBookHome} has C{_path} and L{_addressbookStore} attributes,
        indicating its location on disk and parent store, respectively.
        """
        self.failUnless(
            isinstance(self.home1._path, FilePath),
            self.home1._path
        )
        self.assertEquals(
            self.home1._addressbookStore,
            self.addressbookStore
        )

    def test_addressbookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        name = ".foo"
        self.home1._path.child(name).createDirectory()
        self.assertEquals(self.home1.addressbookWithName(name), None)

    def test_createAddressBookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        self.assertRaises(
            HomeChildNameNotAllowedError,
            self.home1.createAddressBookWithName, ".foo"
        )

    def test_removeAddressBookWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook names may start with ".".
        """
        name = ".foo"
        self.home1._path.child(name).createDirectory()
        self.assertRaises(
            NoSuchHomeChildError,
            self.home1.removeAddressBookWithName, name
        )

class AddressBookTest(unittest.TestCase):

    def setUp(self):
        setUpAddressBook1(self)

    def test_init(self):
        """
        L{AddressBook.__init__} sets private attributes to reflect its constructor
        arguments.
        """
        self.failUnless(
            isinstance(self.addressbook1._path, FilePath),
            self.addressbook1
        )
        self.failUnless(
            isinstance(self.addressbook1._addressbookHome, AddressBookHome),
            self.addressbook1._addressbookHome
        )

    def test_useIndexImmediately(self):
        """
        L{AddressBook._index} is usable in the same transaction it is created, with
        a temporary filename.
        """
        self.home1.createAddressBookWithName("addressbook2")
        addressbook = self.home1.addressbookWithName("addressbook2")
        index = addressbook._index
        self.assertEquals(set(index.addressbookObjects()),
                          set(addressbook.addressbookObjects()))
        self.txn.commit()
        self.txn = self.addressbookStore.newTransaction()
        self.home1 = self.txn.addressbookHomeWithUID("home1")
        addressbook = self.home1.addressbookWithName("addressbook2")
        # FIXME: we should be curating our own index here, but in order to fix
        # that the code in the old implicit scheduler needs to change.  This
        # test would be more effective if there were actually some objects in
        # this list.
        index = addressbook._index
        self.assertEquals(set(index.addressbookObjects()),
                          set(addressbook.addressbookObjects()))

    def test_addressbookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        name = ".foo.vcf"
        self.home1._path.child(name).touch()
        self.assertEquals(self.addressbook1.addressbookObjectWithName(name), None)

    @featureUnimplemented
    def test_addressbookObjectWithUID_exists(self):
        """
        Find existing addressbook object by name.
        """
        addressbookObject = self.addressbook1.addressbookObjectWithUID("1")
        self.failUnless(
            isinstance(addressbookObject, AddressBookObject),
            addressbookObject
        )
        self.assertEquals(
            addressbookObject.component(),
            self.addressbook1.addressbookObjectWithName("1.vcf").component()
        )

    def test_createAddressBookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        self.assertRaises(
            ObjectResourceNameNotAllowedError,
            self.addressbook1.createAddressBookObjectWithName,
            ".foo", VComponent.fromString(vcard4_text)
        )

    @featureUnimplemented
    def test_createAddressBookObjectWithName_uidconflict(self):
        """
        Attempt to create a addressbook object with a conflicting UID
        should raise.
        """
        name = "foo.vcf"
        assert self.addressbook1.addressbookObjectWithName(name) is None
        component = VComponent.fromString(vcard1modified_text)
        self.assertRaises(
            ObjectResourceUIDAlreadyExistsError,
            self.addressbook1.createAddressBookObjectWithName,
            name, component
        )

    def test_removeAddressBookObject_delayedEffect(self):
        """
        Removing a addressbook object should not immediately remove the underlying
        file; it should only be removed upon commit() of the transaction.
        """
        self.addressbook1.removeAddressBookObjectWithName("2.vcf")
        self.failUnless(self.addressbook1._path.child("2.vcf").exists())
        self.txn.commit()
        self.failIf(self.addressbook1._path.child("2.vcf").exists())

    def test_removeAddressBookObjectWithName_dot(self):
        """
        Filenames starting with "." are reserved by this
        implementation, so no addressbook object names may start with
        ".".
        """
        name = ".foo"
        self.addressbook1._path.child(name).touch()
        self.assertRaises(
            NoSuchObjectResourceError,
            self.addressbook1.removeAddressBookObjectWithName, name
        )

    def _refresh(self):
        """
        Re-read the (committed) home1 and addressbook1 objects in a new
        transaction.
        """
        self.txn = self.addressbookStore.newTransaction()
        self.home1 = self.txn.addressbookHomeWithUID("home1")
        self.addressbook1 = self.home1.addressbookWithName("addressbook_1")

    def test_undoCreateAddressBookObject(self):
        """
        If a addressbook object is created as part of a transaction, it will be
        removed if that transaction has to be aborted.
        """
        # Make sure that the addressbook home is actually committed; rolling back
        # addressbook home creation will remove the whole directory.
        self.txn.commit()
        self._refresh()
        self.addressbook1.createAddressBookObjectWithName(
            "sample.vcf",
            VComponent.fromString(vcard4_text)
        )
        self._refresh()
        self.assertIdentical(
            self.addressbook1.addressbookObjectWithName("sample.vcf"),
            None
        )

    def doThenUndo(self):
        """
        Commit the current transaction, but add an operation that will cause it
        to fail at the end.  Finally, refresh all attributes with a new
        transaction so that further oparations can be performed in a valid
        context.
        """
        def fail():
            raise RuntimeError("oops")
        self.txn.addOperation(fail, "dummy failing operation")
        self.assertRaises(RuntimeError, self.txn.commit)
        self._refresh()

    def test_undoModifyAddressBookObject(self):
        """
        If an existing addressbook object is modified as part of a transaction, it
        should be restored to its previous status if the transaction aborts.
        """
        originalComponent = self.addressbook1.addressbookObjectWithName(
            "1.vcf").component()
        self.addressbook1.addressbookObjectWithName("1.vcf").setComponent(
            VComponent.fromString(vcard1modified_text)
        )
        # Sanity check.
        self.assertEquals(
            self.addressbook1.addressbookObjectWithName("1.vcf").component(),
            VComponent.fromString(vcard1modified_text)
        )
        self.doThenUndo()
        self.assertEquals(
            self.addressbook1.addressbookObjectWithName("1.vcf").component(),
            originalComponent
        )

    def test_modifyAddressBookObjectCaches(self):
        """
        Modifying a addressbook object should cache the modified component in
        memory, to avoid unnecessary parsing round-trips.
        """
        modifiedComponent = VComponent.fromString(vcard1modified_text)
        self.addressbook1.addressbookObjectWithName("1.vcf").setComponent(
            modifiedComponent
        )
        self.assertIdentical(
            modifiedComponent,
            self.addressbook1.addressbookObjectWithName("1.vcf").component()
        )

    @featureUnimplemented
    def test_removeAddressBookObjectWithUID_absent(self):
        """
        Attempt to remove an non-existing addressbook object should raise.
        """
        self.assertRaises(
            NoSuchObjectResourceError,
            self.addressbook1.removeAddressBookObjectWithUID, "xyzzy"
        )

    @testUnimplemented
    def test_syncToken(self):
        """
        Sync token is correct.
        """
        raise NotImplementedError()

    @testUnimplemented
    def test_addressbookObjectsInTimeRange(self):
        """
        Find addressbook objects occuring in a given time range.
        """
        raise NotImplementedError()

    @testUnimplemented
    def test_addressbookObjectsSinceToken(self):
        """
        Find addressbook objects that have been modified since a given
        sync token.
        """
        raise NotImplementedError()

class AddressBookObjectTest(unittest.TestCase):
    def setUp(self):
        setUpAddressBook1(self)
        self.object1 = self.addressbook1.addressbookObjectWithName("1.vcf")

    def test_init(self):
        """
        L{AddressBookObject} has instance attributes, C{_path} and C{_addressbook},
        which refer to its position in the filesystem and the addressbook in which
        it is contained, respectively.
        """
        self.failUnless(
            isinstance(self.object1._path, FilePath),
            self.object1._path
        )
        self.failUnless(
            isinstance(self.object1._addressbook, AddressBook),
            self.object1._addressbook
        )

class FileStorageTests(unittest.TestCase, CommonTests):
    """
    File storage tests.
    """

    addressbookStore = None

    def storeUnderTest(self):
        """
        Create and return a L{AddressBookStore} for testing.
        """
        if self.addressbookStore is None:
            setUpAddressBookStore(self)
        return self.addressbookStore

    def test_init(self):
        """
        L{AddressBookStore} has a C{_path} attribute which refers to its
        constructor argument.
        """
        self.assertEquals(self.storeUnderTest()._path,
                          self.storeRootPath)

    def test_addressbookObjectsWithDotFile(self):
        """
        Adding a dotfile to the addressbook home should not increase
        """
        self.homeUnderTest()._path.child(".foo").createDirectory()
        self.test_addressbookObjects()

<|endoftext|>


<commit_before>import logging
import os
import shutil
import sys

class Scraper:
    """Scraper class."""

    logger = logging.getLogger("uoftscrapers")

    @staticmethod
    def ensure_location(location):
        """Ensure that the location given exists."""

        if not os.path.exists(location):
            os.makedirs(location)

    @staticmethod
    def flush_percentage(decimal):
        """Update the last line in stdout to a percentage formatted value."""

        sys.stdout.write('%.2f%%\r' % (decimal * 100))
        sys.stdout.flush()

    @staticmethod
    def get_html(s, url):
        html = None
        while html is None:
            try:
                r = s.get(url)
                if r.status_code == 200:
                    html = r.text
            except (requests.exceptions.Timeout,
                    requests.exceptions.ConnectionError):
                continue

        return html.encode('utf-8')

    @staticmethod
    def get_text_from_class(soup, name):
        obj = soup.find(class_=name)
        if obj != None:
            return obj.get_text().replace('\xa0', ' ').strip()
        else:
            return ''
<commit_msg>Add headers param<commit_after>import logging
import os
import shutil
import sys

class Scraper:
    """Scraper class."""

    logger = logging.getLogger("uoftscrapers")

    @staticmethod
    def ensure_location(location):
        """Ensure that the location given exists."""

        if not os.path.exists(location):
            os.makedirs(location)

    @staticmethod
    def flush_percentage(decimal):
        """Update the last line in stdout to a percentage formatted value."""

        sys.stdout.write('%.2f%%\r' % (decimal * 100))
        sys.stdout.flush()

    @staticmethod
    def get_html(s, url, headers=None):
        """Fetches the HTML page source, automatically retrying if it times out."""

        html = None
        while html is None:
            try:
                r = s.get(url, headers=headers)
                if r.status_code == 200:
                    html = r.text
            except (requests.exceptions.Timeout,
                    requests.exceptions.ConnectionError):
                continue

        return html.encode('utf-8')

    @staticmethod
    def get_text_from_class(soup, name):
        obj = soup.find(class_=name)
        if obj != None:
            return obj.get_text().replace('\xa0', ' ').strip()
        else:
            return ''
<|endoftext|>


<commit_before>#! /bin/env python
import json
import os
import requests
import sys

from configparser import ConfigParser
from txclib import utils
from txclib import config

STATUS_FILE_PATH = '../src/locales/translationStatus.json'
CONFIG_FILE_PATH = '../.tx/config'
SCRIPT_PATH = os.path.dirname(os.path.realpath(sys.argv[0]))
TRANSLATION_URL = 'https://api.transifex.com/organizations/yunity-1/projects/karrot/'
DEFAULT_TOKEN_URL = 'https://www.transifex.com'

STATUS_FILE = os.path.join(SCRIPT_PATH, STATUS_FILE_PATH)
CONFIG_FILE = os.path.join(SCRIPT_PATH, CONFIG_FILE_PATH)

def get_token():
    token = os.environ.get('TX_TOKEN')
    if token:
        return token

    txrc_file = utils.get_transifex_file()
    txrc = config.OrderedRawConfigParser()
    txrc.read((txrc_file,))
    return txrc.get(DEFAULT_TOKEN_URL, 'token')

def get_translation_status(token):
    response = requests.get(TRANSLATION_URL, auth=('api', token))
    if response.status_code != requests.codes.OK:
        print('Failed to get translation status!\n')
        print(response.json())
        sys.exit(1)
    return response.json()['stats']

def map_status_to_percentage(translation_status):
    percentage_dict = {lang: round(v['translated']['percentage']*100) for lang, v in translation_status.items()}
    percentage_dict['en'] = 100
    return percentage_dict

def get_raw_language_map_from_config_file(config_file):
    configuration = ConfigParser()
    configuration.read(config_file)
    lang_map_raw = configuration.get('main', 'lang_map').replace(' ', '').split(',')
    return lang_map_raw

def clean_lang_map_raw(lang_map_raw):
    lang_map = dict()
    for arg in lang_map_raw:
        k, v = arg.split(":")
        lang_map[k] = v
    return lang_map

def translate_key(key, lang_map):
    """Translate to HTTP Accept-Language format"""
    if key in lang_map:
        key = lang_map[key]
    return key.replace('_', '-').lower()

def get_translated_dict(lang_map, percentage_dict):
    for k, v in percentage_dict.items():
        translate_key(k, lang_map)
    return percentage_dict

def write_dict_to_file(dictionary, filename):
    with open(STATUS_FILE, 'w') as f:
        json.dump(dictionary, f, sort_keys=True, indent=2)
        print("Updated", STATUS_FILE)

token = get_token()
translation_status = get_translation_status(token)
percentage_dict = map_status_to_percentage(translation_status)

lang_map_raw = get_raw_language_map_from_config_file(CONFIG_FILE)
lang_map = clean_lang_map_raw(lang_map_raw)

translated_dict = get_translated_dict(lang_map, percentage_dict)

write_dict_to_file(translated_dict, STATUS_FILE)
<commit_msg>Revert "General refactoring (#1790)"<commit_after>#! /bin/env python
import json
from txclib import utils
from txclib import config
import requests
import os
import sys
import configparser

script_path = os.path.dirname(os.path.realpath(sys.argv[0]))

status_file = os.path.join(script_path, '../src/locales/translationStatus.json')
config_file = os.path.join(script_path, '../.tx/config')

def get_token():
    token = os.environ.get('TX_TOKEN')
    if token:
        return token

    txrc_file = utils.get_transifex_file()
    txrc = config.OrderedRawConfigParser()
    txrc.read((txrc_file,))
    return txrc.get('https://www.transifex.com', 'token')

token = get_token()

r = requests.get('https://api.transifex.com/organizations/yunity-1/projects/karrot/',
  auth=('api', token)
)

if r.status_code != 200:
    print('Failed to get translation status!')
    print('Response is:')
    print(r.json())
    sys.exit(1)

stats = r.json()['stats']
percentage_dict = {lang: round(v['translated']['percentage']*100) for lang, v in stats.items()}
percentage_dict['en'] = 100

config = configparser.ConfigParser()
config.read(config_file)
lang_map_raw = config.get('main', 'lang_map')
lang_map = {}
for arg in lang_map_raw.replace(' ', '').split(','):
    k, v = arg.split(":")
    lang_map.update({k: v})

# translate to HTTP Accept-Language format
def translate_key(key):
    if key in lang_map:
        key = lang_map[key]
    return key.replace('_', '-').lower()

translated_dict = { translate_key(k): v for (k, v) in percentage_dict.items() }

with open(status_file, 'w') as f:
    json.dump(translated_dict, f, sort_keys=True, indent=2)
    print("Updated", status_file)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

__author__ = "Yuji Ikeda"

import numpy as np
from upho.analysis.mappings_modifier import MappingsModifier

class ElementWeightsCalculator(object):
    """Extract weights on elements from eigenvectors"""
    def __init__(self, unitcell, primitive):
        """

        Parameters
        ----------
        unitcell : Phonopy Atoms object
            This may have a disordered atomic configuration.
        primitive : Phonopy Primitive object
        """
        self._extract_map_elements(unitcell)
        self._extract_map_atoms_u2p(primitive)

    def _extract_map_elements(self, unitcell):
        natoms_u = unitcell.get_number_of_atoms()
        elements = unitcell.get_chemical_symbols()
        reduced_elements = sorted(set(elements), key=elements.index)

        map_elements = []
        for re in reduced_elements:
            map_elements.append(
                [i for i, v in enumerate(elements) if v == re])

        if sum(len(v) for v in map_elements) != natoms_u:
            raise ValueError("Mapping of elements is failed.")

        self._map_elements = map_elements
        self._reduced_elements = reduced_elements

    def _extract_map_atoms_u2p(self, primitive):
        p2s_map = primitive.get_primitive_to_supercell_map()
        s2p_map = primitive.get_supercell_to_primitive_map()
        natoms_u = len(s2p_map)

        map_atoms_u2p = []
        for iatom_s in p2s_map:
            map_atoms_u2p.append(
                [i for i, v in enumerate(s2p_map) if v == iatom_s])

        if sum(len(v) for v in map_atoms_u2p) != natoms_u:
            raise ValueError("Mapping of atoms_u2p is failed.")

        self._map_atoms_u2p = map_atoms_u2p

    def get_map_elements(self):
        return self._map_elements

    def get_map_atoms_u2p(self):
        return self._map_atoms_u2p

    def get_reduced_elements(self):
        return self._reduced_elements

    def get_number_of_elements(self):
        return len(self._reduced_elements)

    def run_star(self, vectors, ndims=3):
        """

        Parameters
        ----------
        vectors : (narms, natoms_u * ndims, nbands) array
        ndims : Integer
            number of dimensions of the space.

        Returns
        -------
        weights : (narms, nelements, natoms_p, nbands) array
        """
        weights = []
        for vectors_arm in vectors:
            weights_arm = self.run(vectors_arm, ndims)
            weights.append(weights_arm)
        return np.array(weights)

    def run(self, vectors, ndims=3):
        """

        Parameters
        ----------
        vectors : (natoms_u * ndims, nbands) array
        ndims : Integer
            number of dimensions of the space.

        Returns
        -------
        weights : (natoms_p, nelements, nbands) array
        """
        map_atoms_u2p = self._map_atoms_u2p
        map_elements = self._map_elements

        shape = vectors.shape
        nbands = shape[1]
        tmp = vectors.reshape(shape[0] // ndims, ndims, nbands)
        weights_atoms = np.linalg.norm(tmp, axis=1) ** 2

        shape_weights = (len(map_atoms_u2p), len(map_elements), nbands)
        weights = np.full(shape_weights, np.nan)  # Initialization

        for ip, lp in enumerate(map_atoms_u2p):
            for ie, le in enumerate(map_elements):
                indices = sorted(set(lp) & set(le))
                weights[ip, ie] = np.sum(weights_atoms[indices], axis=0)

        return weights

    def project_vectors(self, vectors, ndims=3):
        map_atoms_u2p = self._map_atoms_u2p
        map_elements = self._map_elements

        natoms_p = len(map_atoms_u2p)
        num_elements = len(map_elements)

        tmp = np.zeros_like(vectors[None, None])  # Add two dimensions
        projected_vectors = (
            np.repeat(np.repeat(tmp, natoms_p, axis=0), num_elements, axis=1))

        for ip, lp in enumerate(map_atoms_u2p):
            for ie, le in enumerate(map_elements):
                indices_tmp = sorted(set(lp) & set(le))
                indices = MappingsModifier(indices_tmp).expand_mappings(ndims)
                projected_vectors[ip, ie, indices] = vectors[indices]

        return projected_vectors
<commit_msg>Fix not to give "None" indices when the element "le" does not exist on the sublattice.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

__author__ = "Yuji Ikeda"

import numpy as np
from upho.analysis.mappings_modifier import MappingsModifier

class ElementWeightsCalculator(object):
    """Extract weights on elements from eigenvectors"""
    def __init__(self, unitcell, primitive):
        """

        Parameters
        ----------
        unitcell : Phonopy Atoms object
            This may have a disordered atomic configuration.
        primitive : Phonopy Primitive object
        """
        self._extract_map_elements(unitcell)
        self._extract_map_atoms_u2p(primitive)

    def _extract_map_elements(self, unitcell):
        natoms_u = unitcell.get_number_of_atoms()
        elements = unitcell.get_chemical_symbols()
        reduced_elements = sorted(set(elements), key=elements.index)

        map_elements = []
        for re in reduced_elements:
            map_elements.append(
                [i for i, v in enumerate(elements) if v == re])

        if sum(len(v) for v in map_elements) != natoms_u:
            raise ValueError("Mapping of elements is failed.")

        self._map_elements = map_elements
        self._reduced_elements = reduced_elements

    def _extract_map_atoms_u2p(self, primitive):
        p2s_map = primitive.get_primitive_to_supercell_map()
        s2p_map = primitive.get_supercell_to_primitive_map()
        natoms_u = len(s2p_map)

        map_atoms_u2p = []
        for iatom_s in p2s_map:
            map_atoms_u2p.append(
                [i for i, v in enumerate(s2p_map) if v == iatom_s])

        if sum(len(v) for v in map_atoms_u2p) != natoms_u:
            raise ValueError("Mapping of atoms_u2p is failed.")

        self._map_atoms_u2p = map_atoms_u2p

    def get_map_elements(self):
        return self._map_elements

    def get_map_atoms_u2p(self):
        return self._map_atoms_u2p

    def get_reduced_elements(self):
        return self._reduced_elements

    def get_number_of_elements(self):
        return len(self._reduced_elements)

    def run_star(self, vectors, ndims=3):
        """

        Parameters
        ----------
        vectors : (narms, natoms_u * ndims, nbands) array
        ndims : Integer
            number of dimensions of the space.

        Returns
        -------
        weights : (narms, nelements, natoms_p, nbands) array
        """
        weights = []
        for vectors_arm in vectors:
            weights_arm = self.run(vectors_arm, ndims)
            weights.append(weights_arm)
        return np.array(weights)

    def run(self, vectors, ndims=3):
        """

        Parameters
        ----------
        vectors : (natoms_u * ndims, nbands) array
        ndims : Integer
            number of dimensions of the space.

        Returns
        -------
        weights : (natoms_p, nelements, nbands) array
        """
        map_atoms_u2p = self._map_atoms_u2p
        map_elements = self._map_elements

        shape = vectors.shape
        nbands = shape[1]
        tmp = vectors.reshape(shape[0] // ndims, ndims, nbands)
        weights_atoms = np.linalg.norm(tmp, axis=1) ** 2

        shape_weights = (len(map_atoms_u2p), len(map_elements), nbands)
        weights = np.full(shape_weights, np.nan)  # Initialization

        for ip, lp in enumerate(map_atoms_u2p):
            for ie, le in enumerate(map_elements):
                indices = sorted(set(lp) & set(le))
                weights[ip, ie] = np.sum(weights_atoms[indices], axis=0)

        return weights

    def project_vectors(self, vectors, ndims=3):
        map_atoms_u2p = self._map_atoms_u2p
        map_elements = self._map_elements

        natoms_p = len(map_atoms_u2p)
        num_elements = len(map_elements)

        tmp = np.zeros_like(vectors[None, None])  # Add two dimensions
        projected_vectors = (
            np.repeat(np.repeat(tmp, natoms_p, axis=0), num_elements, axis=1))

        for ip, lp in enumerate(map_atoms_u2p):
            for ie, le in enumerate(map_elements):
                indices_tmp = sorted(set(lp) & set(le))
                indices = MappingsModifier(indices_tmp).expand_mappings(ndims)
                if len(indices) > 0:  # The element "le" exists on the sublattice.
                    projected_vectors[ip, ie, indices] = vectors[indices]

        return projected_vectors
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# import json
import boto3
from dcicutils import ff_utils
from core.utils import powerup
from core.utils import TibannaStartException
from core.utils import printlog
from core.pony_utils import (
    Tibanna,
    merge_source_experiments,
    create_ffmeta_awsem,
    aslist,
    ProcessedFileMetadata,
    FormatExtensionMap,
    get_extra_file_key,
    create_ffmeta_input_files_from_pony_input_file_list
)
import random

s3 = boto3.resource('s3')

def metadata_only(event):
    # this relies on the fact that event contains and output key with output files
    assert event['metadata_only']
    assert event['output_files']
    return real_handler(event, None)

@powerup('start_run_awsem', metadata_only)
def handler(event, context):
    if event.get('push_error_to_end', True):
        event['push_error_to_end'] = True  # push error to end by default for pony
    return real_handler(event, context)

def real_handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    for infile in input_file_list:
        if not infile:
            raise("malformed input, check your input_files")
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    parameters = ff_utils.convert_param(event.get('parameters'), True)
    tibanna_settings = event.get('_tibanna', {})
    tag = event.get('tag')
    # if they don't pass in env guess it from output_bucket
    try:
        env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
        # tibanna provides access to keys based on env and stuff like that
        tibanna = Tibanna(env, ff_keys=event.get('ff_keys'), settings=tibanna_settings)
    except Exception as e:
        raise TibannaStartException("%s" % e)

    args = dict()

    # get argument format & type info from workflow
    wf_meta = ff_utils.get_metadata(workflow_uuid,
                                    key=tibanna.ff_keys,
                                    ff_env=tibanna.env,
                                    add_on='frame=object')
    printlog("workflow info  %s" % wf_meta)
    if 'error' in wf_meta.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for %s fourfront" % workflow_uuid)

    # get cwl info from wf_meta
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        printlog(wf_meta.get(k))
        args[k] = wf_meta.get(k)
    if not args['cwl_child_filenames']:
        args['cwl_child_filenames'] = []

    # switch to v1 if available
    if 'cwl_directory_url_v1' in wf_meta:  # use CWL v1
        args['cwl_directory_url'] = wf_meta['cwl_directory_url_v1']
        args['cwl_version'] = 'v1'
    else:
        args['cwl_version'] = 'draft3'

    # input file args for awsem
    for input_file in input_file_list:
        process_input_file_info(input_file, tibanna.ff_keys, tibanna.env, args)

    # create the ff_meta output info
    input_files_for_ffmeta = create_ffmeta_input_files_from_pony_input_file_list(input_file_list)

    # source experiments
    input_file_uuids = [_['uuid'] for _ in input_file_list]
    pf_source_experiments = merge_source_experiments(input_file_uuids,
                                                     tibanna.ff_keys,
                                                     tibanna.env)

    # processed file metadata
    output_files, pf_meta = \
        create_wfr_output_files_and_processed_files(wf_meta, tibanna,
                                                    pf_source_experiments,
                                                    custom_fields=event.get('custom_pf_fields'),
                                                    user_supplied_output_files=event.get('output_files'))
    print("output files= %s" % str(output_files))

    # 4DN dcic award and lab are used here, unless provided in wfr_meta
    ff_meta = create_ffmeta_awsem(
        workflow_uuid, app_name, input_files_for_ffmeta, tag=tag,
        run_url=tibanna.settings.get('url', ''),
        output_files=output_files, parameters=parameters,
        extra_meta=event.get('wfr_meta'),
    )

    printlog("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post(key=tibanna.ff_keys)

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    args['secondary_output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            random_tag = str(int(random.random() * 1000000000000))
            # add a random tag at the end for non-processed file e.g. md5 report,
            # so that if two or more wfr are trigerred (e.g. one with parent file, one with extra file)
            # it will create a different output. Not implemented for processed files -
            # it's tricky because processed files must have a specific name.
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name + random_tag
        if 'secondary_file_formats' in of and 'extra_files' in of:
            for ext in of.get('extra_files'):
                args['secondary_output_target'][arg_name].append(ext.get('upload_key'))

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # dependencies
    if 'dependency' in event:
        args['dependency'] = event['dependency']

    # initialize config parameters as null for benchmarking
    config = event['config']
    if 'instance_type' not in config:
        config['instance_type'] = ''
    if 'EBS_optimized' not in config:
        config['EBS_optimized'] = ''
    if 'ebs_size' not in config:
        config['ebs_size'] = 0
    if 'public_postrun_json' not in config:
        config['public_postrun_json'] = True

    event.update({"ff_meta": ff_meta.as_dict(),
                  'pf_meta': [meta.as_dict() for meta in pf_meta],
                  "_tibanna": tibanna.as_dict(),
                  "args": args
                  })
    return(event)

def process_input_file_info(input_file, ff_keys, ff_env, args):
    if not args or 'input_files' not in args:
        args['input_files'] = dict()
    if not args or 'secondary_files' not in args:
        args['secondary_files'] = dict()
    if isinstance(input_file['uuid'], unicode):
        input_file['uuid'] = input_file['uuid'].encode('utf-8')
    if isinstance(input_file['object_key'], unicode):
        input_file['object_key'] = input_file['object_key'].encode('utf-8')
    if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
        object_key = input_file['uuid'] + '/' + input_file['object_key']
    elif (isinstance(input_file['uuid'], list) and
          isinstance(input_file['object_key'], list) and
          len(input_file['uuid']) == len(input_file['object_key'])):

        object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
    else:
        raise Exception("input_file uuid and object_key should match in their type and length (if lists) : " +
                        "type{}{} length{}{}".format(type(input_file['uuid']), type(input_file['object_key']),
                                                     len(input_file['uuid']), len(input_file['object_key'])))
    args['input_files'].update({input_file['workflow_argument_name']: {
                                'bucket_name': input_file['bucket_name'],
                                'object_key': object_key}})
    if input_file.get('format_if_extra', ''):
        args['input_files'][input_file['workflow_argument_name']]['format_if_extra'] \
            = input_file.get('format_if_extra')
    else:  # do not add this if the input itself is an extra file
        add_secondary_files_to_args(input_file, ff_keys, ff_env, args)

def add_secondary_files_to_args(input_file, ff_keys, ff_env, args):
    if not args or 'input_files' not in args:
        raise Exception("args must contain key 'input_files'")
    if 'secondary_files'not in args:
        args['secondary_files'] = dict()
    inf_uuids = aslist(input_file['uuid'])
    argname = input_file['workflow_argument_name']
    extra_file_keys = []
    inf_object_key = args['input_files'][argname]['object_key']
    inf_keys = aslist(inf_object_key)
    fe_map = None
    for i, inf_uuid in enumerate(inf_uuids):
        infile_meta = ff_utils.get_metadata(inf_uuid,
                                            key=ff_keys,
                                            ff_env=ff_env,
                                            add_on='frame=object')
        if infile_meta.get('extra_files'):
            infile_format = infile_meta.get('file_format')
            infile_key = inf_keys[i]
            if not fe_map:
                fe_map = FormatExtensionMap(ff_keys)
            for extra_file in infile_meta.get('extra_files'):
                extra_file_format = extra_file.get('file_format')
                extra_file_key = get_extra_file_key(infile_format, infile_key, extra_file_format, fe_map)
                extra_file_keys.append(extra_file_key)

    if len(extra_file_keys) > 0:
        if len(extra_file_keys) == 1:
            extra_file_keys = extra_file_keys[0]
        args['secondary_files'].update({input_file['workflow_argument_name']: {
                                        'bucket_name': input_file['bucket_name'],
                                        'object_key': extra_file_keys}})

def user_supplied_proc_file(user_supplied_output_files, arg_name, tibanna):
    if not user_supplied_output_files:
        raise Exception("user supplied processed files missing\n")
    of = [output for output in user_supplied_output_files if output.get('workflow_argument_name') == arg_name]
    if of:
        if len(of) > 1:
            raise Exception("multiple output files supplied with same workflow_argument_name")
        of = of[0]
        return ProcessedFileMetadata.get(of.get('uuid'), tibanna.ff_keys,
                                         tibanna.env, return_data=True)
    else:
        printlog("no output_files found in input_json matching arg_name")
        printlog("user_supplied_output_files: %s" % str(user_supplied_output_files))
        printlog("arg_name: %s" % str(arg_name))
        printlog("tibanna is %s" % str(tibanna))
        raise Exception("user supplied processed files missing\n")

class WorkflowRunOutputFiles(object):
    def __init__(self, workflow_argument_name, argument_type, file_format=None, secondary_file_formats=None,
                 upload_key=None, uuid=None, extra_files=None):
        self.workflow_argument_name = workflow_argument_name
        self.type = argument_type
        self.format = file_format
        self.secondary_file_formats = secondary_file_formats
        self.value = uuid
        self.upload_key = upload_key
        self.extra_files = extra_files

    def as_dict(self):
        return self.__dict__

def parse_custom_fields(custom_fields, argname):
    pf_other_fields = dict()
    if custom_fields:
        if argname in custom_fields:
            pf_other_fields.update(custom_fields[argname])
        if 'ALL' in custom_fields:
            pf_other_fields.update(custom_fields['ALL'])
    return pf_other_fields

def create_and_post_processed_file(ff_keys, file_format, secondary_file_formats,
                                   source_experiments=None, other_fields=None):
    if not file_format:
        raise Exception("file format for processed file must be provided")
    extra_files = [{"file_format": v} for v in secondary_file_formats]
    pf = ProcessedFileMetadata(
        file_format=file_format,
        extra_files=extra_files,
        source_experiments=source_experiments,
        other_fields=other_fields
    )
    # actually post processed file metadata here
    resp = pf.post(key=ff_keys)
    if resp and '@graph' in resp:
        resp = resp.get('@graph')[0]
    else:
        raise Exception("Failed to post Processed file metadata.\n")
    return pf, resp

def create_wfr_outputfiles(arg, resp):
    return WorkflowRunOutputFiles(arg.get('workflow_argument_name'), arg.get('argument_type'),
                                  arg.get('argument_format', None), arg.get('secondary_file_formats', None),
                                  resp.get('upload_key', None), resp.get('uuid', None), resp.get('extra_files', None))

def create_wfr_output_files_and_processed_files(wf_meta, tibanna, pf_source_experiments=None,
                                                custom_fields=None, user_supplied_output_files=None):
    output_files = []
    pf_meta = []
    arg_type_list = ['Output processed file', 'Output report file', 'Output QC file']
    for arg in wf_meta.get('arguments', []):
        printlog("processing arguments %s" % str(arg))
        if arg.get('argument_type') in arg_type_list:
            if user_supplied_output_files:
                pf, resp = user_supplied_proc_file(user_supplied_output_files,
                                                   arg.get('workflow_argument_name'),
                                                   tibanna)
                printlog("proc_file_for_arg_name returned %s \nfrom ff result of\n %s" % (str(pf.__dict__), str(resp)))
            else:
                if arg.get('argument_type', '') == 'Output processed file':
                    argname = arg.get('workflow_argument_name')
                    pf, resp = create_and_post_processed_file(tibanna.ff_keys,
                                                              arg.get('argument_format', ''),
                                                              arg.get('secondary_file_formats', []),
                                                              pf_source_experiments,
                                                              parse_custom_fields(custom_fields, argname))
                else:
                    pf = None
                    resp = dict()
            of = create_wfr_outputfiles(arg, resp)
            if pf:
                pf_meta.append(pf)
            if of:
                output_files.append(of.as_dict())
    return output_files, pf_meta
<commit_msg>case with no extra file minor fix<commit_after># -*- coding: utf-8 -*-
# import json
import boto3
from dcicutils import ff_utils
from core.utils import powerup
from core.utils import TibannaStartException
from core.utils import printlog
from core.pony_utils import (
    Tibanna,
    merge_source_experiments,
    create_ffmeta_awsem,
    aslist,
    ProcessedFileMetadata,
    FormatExtensionMap,
    get_extra_file_key,
    create_ffmeta_input_files_from_pony_input_file_list
)
import random

s3 = boto3.resource('s3')

def metadata_only(event):
    # this relies on the fact that event contains and output key with output files
    assert event['metadata_only']
    assert event['output_files']
    return real_handler(event, None)

@powerup('start_run_awsem', metadata_only)
def handler(event, context):
    if event.get('push_error_to_end', True):
        event['push_error_to_end'] = True  # push error to end by default for pony
    return real_handler(event, context)

def real_handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    for infile in input_file_list:
        if not infile:
            raise("malformed input, check your input_files")
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    parameters = ff_utils.convert_param(event.get('parameters'), True)
    tibanna_settings = event.get('_tibanna', {})
    tag = event.get('tag')
    # if they don't pass in env guess it from output_bucket
    try:
        env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
        # tibanna provides access to keys based on env and stuff like that
        tibanna = Tibanna(env, ff_keys=event.get('ff_keys'), settings=tibanna_settings)
    except Exception as e:
        raise TibannaStartException("%s" % e)

    args = dict()

    # get argument format & type info from workflow
    wf_meta = ff_utils.get_metadata(workflow_uuid,
                                    key=tibanna.ff_keys,
                                    ff_env=tibanna.env,
                                    add_on='frame=object')
    printlog("workflow info  %s" % wf_meta)
    if 'error' in wf_meta.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for %s fourfront" % workflow_uuid)

    # get cwl info from wf_meta
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        printlog(wf_meta.get(k))
        args[k] = wf_meta.get(k)
    if not args['cwl_child_filenames']:
        args['cwl_child_filenames'] = []

    # switch to v1 if available
    if 'cwl_directory_url_v1' in wf_meta:  # use CWL v1
        args['cwl_directory_url'] = wf_meta['cwl_directory_url_v1']
        args['cwl_version'] = 'v1'
    else:
        args['cwl_version'] = 'draft3'

    # input file args for awsem
    for input_file in input_file_list:
        process_input_file_info(input_file, tibanna.ff_keys, tibanna.env, args)

    # create the ff_meta output info
    input_files_for_ffmeta = create_ffmeta_input_files_from_pony_input_file_list(input_file_list)

    # source experiments
    input_file_uuids = [_['uuid'] for _ in input_file_list]
    pf_source_experiments = merge_source_experiments(input_file_uuids,
                                                     tibanna.ff_keys,
                                                     tibanna.env)

    # processed file metadata
    output_files, pf_meta = \
        create_wfr_output_files_and_processed_files(wf_meta, tibanna,
                                                    pf_source_experiments,
                                                    custom_fields=event.get('custom_pf_fields'),
                                                    user_supplied_output_files=event.get('output_files'))
    print("output files= %s" % str(output_files))

    # 4DN dcic award and lab are used here, unless provided in wfr_meta
    ff_meta = create_ffmeta_awsem(
        workflow_uuid, app_name, input_files_for_ffmeta, tag=tag,
        run_url=tibanna.settings.get('url', ''),
        output_files=output_files, parameters=parameters,
        extra_meta=event.get('wfr_meta'),
    )

    printlog("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post(key=tibanna.ff_keys)

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    args['secondary_output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            random_tag = str(int(random.random() * 1000000000000))
            # add a random tag at the end for non-processed file e.g. md5 report,
            # so that if two or more wfr are trigerred (e.g. one with parent file, one with extra file)
            # it will create a different output. Not implemented for processed files -
            # it's tricky because processed files must have a specific name.
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name + random_tag
        if 'secondary_file_formats' in of and 'extra_files' in of and of['extra_files']:
            for ext in of.get('extra_files'):
                args['secondary_output_target'][arg_name].append(ext.get('upload_key'))

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # dependencies
    if 'dependency' in event:
        args['dependency'] = event['dependency']

    # initialize config parameters as null for benchmarking
    config = event['config']
    if 'instance_type' not in config:
        config['instance_type'] = ''
    if 'EBS_optimized' not in config:
        config['EBS_optimized'] = ''
    if 'ebs_size' not in config:
        config['ebs_size'] = 0
    if 'public_postrun_json' not in config:
        config['public_postrun_json'] = True

    event.update({"ff_meta": ff_meta.as_dict(),
                  'pf_meta': [meta.as_dict() for meta in pf_meta],
                  "_tibanna": tibanna.as_dict(),
                  "args": args
                  })
    return(event)

def process_input_file_info(input_file, ff_keys, ff_env, args):
    if not args or 'input_files' not in args:
        args['input_files'] = dict()
    if not args or 'secondary_files' not in args:
        args['secondary_files'] = dict()
    if isinstance(input_file['uuid'], unicode):
        input_file['uuid'] = input_file['uuid'].encode('utf-8')
    if isinstance(input_file['object_key'], unicode):
        input_file['object_key'] = input_file['object_key'].encode('utf-8')
    if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
        object_key = input_file['uuid'] + '/' + input_file['object_key']
    elif (isinstance(input_file['uuid'], list) and
          isinstance(input_file['object_key'], list) and
          len(input_file['uuid']) == len(input_file['object_key'])):

        object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
    else:
        raise Exception("input_file uuid and object_key should match in their type and length (if lists) : " +
                        "type{}{} length{}{}".format(type(input_file['uuid']), type(input_file['object_key']),
                                                     len(input_file['uuid']), len(input_file['object_key'])))
    args['input_files'].update({input_file['workflow_argument_name']: {
                                'bucket_name': input_file['bucket_name'],
                                'object_key': object_key}})
    if input_file.get('format_if_extra', ''):
        args['input_files'][input_file['workflow_argument_name']]['format_if_extra'] \
            = input_file.get('format_if_extra')
    else:  # do not add this if the input itself is an extra file
        add_secondary_files_to_args(input_file, ff_keys, ff_env, args)

def parse_formatstr(file_format_str):
    return file_format_str.replace('/file-formats/', '').replace('/', '')

def add_secondary_files_to_args(input_file, ff_keys, ff_env, args):
    if not args or 'input_files' not in args:
        raise Exception("args must contain key 'input_files'")
    if 'secondary_files'not in args:
        args['secondary_files'] = dict()
    inf_uuids = aslist(input_file['uuid'])
    argname = input_file['workflow_argument_name']
    extra_file_keys = []
    inf_object_key = args['input_files'][argname]['object_key']
    inf_keys = aslist(inf_object_key)
    fe_map = None
    for i, inf_uuid in enumerate(inf_uuids):
        infile_meta = ff_utils.get_metadata(inf_uuid,
                                            key=ff_keys,
                                            ff_env=ff_env,
                                            add_on='frame=object')
        if infile_meta.get('extra_files'):
            infile_format = parse_formatstr(infile_meta.get('file_format'))
            infile_key = inf_keys[i]
            if not fe_map:
                fe_map = FormatExtensionMap(ff_keys)
            for extra_file in infile_meta.get('extra_files'):
                extra_file_format = parse_formatstr(extra_file.get('file_format'))
                extra_file_key = get_extra_file_key(infile_format, infile_key, extra_file_format, fe_map)
                extra_file_keys.append(extra_file_key)

    if len(extra_file_keys) > 0:
        if len(extra_file_keys) == 1:
            extra_file_keys = extra_file_keys[0]
        args['secondary_files'].update({input_file['workflow_argument_name']: {
                                        'bucket_name': input_file['bucket_name'],
                                        'object_key': extra_file_keys}})

def user_supplied_proc_file(user_supplied_output_files, arg_name, tibanna):
    if not user_supplied_output_files:
        raise Exception("user supplied processed files missing\n")
    of = [output for output in user_supplied_output_files if output.get('workflow_argument_name') == arg_name]
    if of:
        if len(of) > 1:
            raise Exception("multiple output files supplied with same workflow_argument_name")
        of = of[0]
        return ProcessedFileMetadata.get(of.get('uuid'), tibanna.ff_keys,
                                         tibanna.env, return_data=True)
    else:
        printlog("no output_files found in input_json matching arg_name")
        printlog("user_supplied_output_files: %s" % str(user_supplied_output_files))
        printlog("arg_name: %s" % str(arg_name))
        printlog("tibanna is %s" % str(tibanna))
        raise Exception("user supplied processed files missing\n")

class WorkflowRunOutputFiles(object):
    def __init__(self, workflow_argument_name, argument_type, file_format=None, secondary_file_formats=None,
                 upload_key=None, uuid=None, extra_files=None):
        self.workflow_argument_name = workflow_argument_name
        self.type = argument_type
        self.format = file_format
        self.secondary_file_formats = secondary_file_formats
        self.value = uuid
        self.upload_key = upload_key
        self.extra_files = extra_files

    def as_dict(self):
        return self.__dict__

def parse_custom_fields(custom_fields, argname):
    pf_other_fields = dict()
    if custom_fields:
        if argname in custom_fields:
            pf_other_fields.update(custom_fields[argname])
        if 'ALL' in custom_fields:
            pf_other_fields.update(custom_fields['ALL'])
    return pf_other_fields

def create_and_post_processed_file(ff_keys, file_format, secondary_file_formats,
                                   source_experiments=None, other_fields=None):
    if not file_format:
        raise Exception("file format for processed file must be provided")
    extra_files = [{"file_format": v} for v in secondary_file_formats]
    pf = ProcessedFileMetadata(
        file_format=file_format,
        extra_files=extra_files,
        source_experiments=source_experiments,
        other_fields=other_fields
    )
    # actually post processed file metadata here
    resp = pf.post(key=ff_keys)
    if resp and '@graph' in resp:
        resp = resp.get('@graph')[0]
    else:
        raise Exception("Failed to post Processed file metadata.\n")
    return pf, resp

def create_wfr_outputfiles(arg, resp):
    return WorkflowRunOutputFiles(arg.get('workflow_argument_name'), arg.get('argument_type'),
                                  arg.get('argument_format', None), arg.get('secondary_file_formats', None),
                                  resp.get('upload_key', None), resp.get('uuid', None), resp.get('extra_files', None))

def create_wfr_output_files_and_processed_files(wf_meta, tibanna, pf_source_experiments=None,
                                                custom_fields=None, user_supplied_output_files=None):
    output_files = []
    pf_meta = []
    arg_type_list = ['Output processed file', 'Output report file', 'Output QC file']
    for arg in wf_meta.get('arguments', []):
        printlog("processing arguments %s" % str(arg))
        if arg.get('argument_type') in arg_type_list:
            if user_supplied_output_files:
                pf, resp = user_supplied_proc_file(user_supplied_output_files,
                                                   arg.get('workflow_argument_name'),
                                                   tibanna)
                printlog("proc_file_for_arg_name returned %s \nfrom ff result of\n %s" % (str(pf.__dict__), str(resp)))
            else:
                if arg.get('argument_type', '') == 'Output processed file':
                    argname = arg.get('workflow_argument_name')
                    pf, resp = create_and_post_processed_file(tibanna.ff_keys,
                                                              arg.get('argument_format', ''),
                                                              arg.get('secondary_file_formats', []),
                                                              pf_source_experiments,
                                                              parse_custom_fields(custom_fields, argname))
                else:
                    pf = None
                    resp = dict()
            of = create_wfr_outputfiles(arg, resp)
            if pf:
                pf_meta.append(pf)
            if of:
                output_files.append(of.as_dict())
    return output_files, pf_meta
<|endoftext|>