<commit_before>#!/usr/bin/python2 -OO
from __future__ import print_function
import argparse
import ConfigParser
import csv
import datetime
import errno
import os
import shutil
import sys
import traceback
import uuid

import transcoder

sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import databaseFunctions
import databaseInterface
import fileOperations
from dicts import ReplacementDict

path = '/usr/share/archivematica/dashboard'
if path not in sys.path:
    sys.path.append(path)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings.common'
from fpr.models import FPRule
from main.models import FileFormatVersion, File

# Return codes
SUCCESS = 0
RULE_FAILED = 1
NO_RULE_FOUND = 2

def get_replacement_dict(opts):
    """ Generates values for all knows %var% replacement variables. """
    prefix = ""
    postfix = ""
    output_dir = ""
    #get file name and extension
    (directory, basename) = os.path.split(opts.file_path)
    directory += os.path.sep  # All paths should have trailing /
    (filename, extension_dot) = os.path.splitext(basename)

    if "preservation" in opts.purpose:
        postfix = "-" + opts.task_uuid
        output_dir = directory
    elif "access" in opts.purpose:
        prefix = opts.file_uuid + "-"
        output_dir = os.path.join(opts.sip_path, "DIP", "objects") + os.path.sep
    elif "thumbnail" in opts.purpose:
        output_dir = os.path.join(opts.sip_path, "thumbnails") + os.path.sep
        postfix = opts.file_uuid
    else:
        print("Unsupported command purpose", opts.purpose, file=sys.stderr)
        return None

    # Populates the standard set of unit variables, so,
    # e.g., %fileUUID% is available
    standard_values = ReplacementDict.frommodel(type_='file',
                                                file_=opts.file_uuid)

    output_filename = ''.join([prefix, filename, postfix])
    replacement_dict = {
        "%outputDirectory%": output_dir,
        "%prefix%": prefix,
        "%postfix%": postfix,
        "%outputFileName%": output_filename, # does not include extension
        "%outputFilePath%": os.path.join(output_dir, output_filename) # does not include extension
    }
    replacement_dict.update(standard_values)
    return replacement_dict

def check_manual_normalization(opts):
    """ Checks for manually normalized file, returns that path or None.

    Checks by looking for access/preservation files for a give original file.

    Check the manualNormalization/access and manualNormalization/preservation
    directories for access and preservation files.  If a nomalization.csv
    file is specified, check there first for the mapping between original
    file and access/preservation file. """

    # If normalization.csv provided, check there for mapping from original
    # to access/preservation file
    normalization_csv = os.path.join(opts.sip_path, "objects", "manualNormalization", "normalization.csv")
    dirname, bname = os.path.split(opts.file_path)
    if os.path.isfile(normalization_csv):
        found = False
        with open(normalization_csv, 'rb') as csv_file:
            reader = csv.reader(csv_file)
            # Search the file for an original filename that matches the one provided
            try:
                for row in reader:
                    if "#" in row[0]: # ignore comments
                        continue
                    original, access_file, preservation_file = row
                    if original.lower() == bname.lower():
                        found = True
                        break
            except csv.Error:
                print("Error reading", normalization_csv, " on line", reader.line_num, file=sys.stderr)
                traceback.print_exc(file=sys.stderr)
                return None

        # If we didn't find a match, let it fall through to the usual method
        if found:
            # No manually normalized file for command classification
            if "preservation" in opts.purpose and not preservation_file:
                return None
            if "access" in opts.purpose and not access_file:
                return None

            # If we found a match, verify access/preservation exists in DB
            # match and pull original location b/c sanitization
            if "preservation" in opts.purpose:
                filename = preservation_file
            elif "access" in opts.purpose:
                filename = access_file
            else:
                return None
            print('Looking for', filename, 'in database')
            # FIXME: SQL uses removedtime=0. Convince Django to express this
            return File.objects.get(sip=opts.sip_uuid, originallocation__iendswith=filename) #removedtime = 0

    # Assume that any access/preservation file found with the right
    # name is the correct one
    bname, _ = os.path.splitext(bname)
    dirname = dirname.replace(opts.sip_path, '%SIPDirectory%')
    path = os.path.join(dirname, bname)
    if "preservation" in opts.purpose:
        path = path.replace("%SIPDirectory%objects/",
            "%SIPDirectory%objects/manualNormalization/preservation/")
    elif "access" in opts.purpose:
        path = path.replace("%SIPDirectory%objects/",
            "%SIPDirectory%objects/manualNormalization/access/")
    else:
        return None
    try:
        # FIXME: SQL uses removedtime=0. Cannot get Django to express this
        return File.objects.get(sip=opts.sip_uuid, currentlocation__startswith=path) #removedtime = 0
    except File.DoesNotExist, File.MultipleObjectsReturned:
        # No file with the correct path found, assume not manually normalized
        return None
    return None

def once_normalized(command, opts, replacement_dict):
    """ Updates the database if normalization completed successfully.

    Callback from transcoder.Command

    For preservation files, adds a normalization event, and derivation, as well
    as updating the size and checksum for the new file in the DB.  Adds format
    information for use in the METS file to FilesIDs.
    """
    transcoded_files = []
    if not command.output_location:
        command.output_location = ""
    if os.path.isfile(command.output_location):
        transcoded_files.append(command.output_location)
    elif os.path.isdir(command.output_location):
        for w in os.walk(command.output_location):
            path, _, files = w
            for p in files:
                p = os.path.join(path, p)
                if os.path.isfile(p):
                    transcoded_files.append(p)
    elif command.output_location:
        print("Error - output file does not exist [", command.output_location, "]", file=sys.stderr)
        command.exit_code = -2

    derivation_event_uuid = str(uuid.uuid4())
    event_detail_output = 'ArchivematicaFPRCommandID="{}"'.format(command.fpcommand.uuid)
    if command.event_detail_command is not None:
        event_detail_output += '; {}'.format(command.event_detail_command.std_out)
    for ef in transcoded_files:
        if "preservation" in opts.purpose:
            today = str(datetime.date.today())
            output_file_uuid = opts.task_uuid # File UUID is the same as task UUID for preservation
            # TODO Add manual normalization for files of same name mapping?
            #Add the new file to the SIP
            path_relative_to_sip = ef.replace(opts.sip_path, "%SIPDirectory%", 1)
            fileOperations.addFileToSIP(
                path_relative_to_sip,
                output_file_uuid, # File UUID
                opts.sip_uuid, # SIP UUID
                opts.task_uuid, # Task UUID
                today, # Current date
                sourceType="creation",
                use="preservation",
            )

            #Calculate new file checksum
            fileOperations.updateSizeAndChecksum(
                output_file_uuid, # File UUID, same as task UUID for preservation
                ef, # File path
                today, # Date
                str(uuid.uuid4()), # Event UUID, new UUID
            )

            # Add derivation link and associated event
            insert_derivation_event(
                original_uuid=opts.file_uuid,
                output_uuid=output_file_uuid,
                derivation_uuid=derivation_event_uuid,
                event_detail_output=event_detail_output,
                outcome_detail_note=path_relative_to_sip,
                today=today,
            )

            sql = "INSERT INTO FilesIDs (fileUUID, formatName, formatVersion, formatRegistryName, formatRegistryKey) VALUES ('%s', '%s', NULL, NULL, NULL);" % (output_file_uuid, command.fpcommand.output_format.description)
            databaseInterface.runSQL(sql)

def insert_derivation_event(original_uuid, output_uuid, derivation_uuid,
        event_detail_output, outcome_detail_note, today=None):
    """ Add the derivation link for preservation files and the event. """
    if today is None:
        today = str(datetime.date.today())
    # Add event information to current file
    databaseFunctions.insertIntoEvents(
       fileUUID=original_uuid,
       eventIdentifierUUID=derivation_uuid,
       eventType="normalization",
       eventDateTime=today,
       eventDetail=event_detail_output,
       eventOutcome="",
       eventOutcomeDetailNote=outcome_detail_note or "",
    )

    # Add linking information between files
    databaseFunctions.insertIntoDerivations(
        sourceFileUUID=original_uuid,
        derivedFileUUID=output_uuid,
        relatedEventUUID=derivation_uuid,
    )

def main(opts):
    """ Find and execute normalization commands on input file. """
    # TODO fix for maildir working only on attachments

    # Find the file and it's FormatVersion (file identification)
    try:
        file_ = File.objects.get(uuid=opts.file_uuid)
    except File.DoesNotExist:
        print('File with uuid', opts.file_uuid, 'does not exist in database.', file=sys.stderr)
        return NO_RULE_FOUND
    print('File found:', file_.uuid, file_.currentlocation)

    # Unless normalization file group use is submissionDocumentation, skip the
    # submissionDocumentation directory
    if opts.normalize_file_grp_use != "submissionDocumentation" and file_.currentlocation.startswith('%SIPDirectory%objects/submissionDocumentation'):
        print('File', os.path.basename(opts.file_path), 'in objects/submissionDocumentation, skipping')
        return SUCCESS

    # Only normalize files where the file's group use and normalize group use match
    if file_.filegrpuse != opts.normalize_file_grp_use:
        print(os.path.basename(opts.file_path), 'is file group usage', file_.filegrpuse, 'instead of ', opts.normalize_file_grp_use, ' - skipping')
        return SUCCESS

    # If a file has been manually normalized for this purpose, skip it
    manually_normalized_file = check_manual_normalization(opts)
    if manually_normalized_file:
        print(os.path.basename(opts.file_path), 'was already manually normalized into', manually_normalized_file.currentlocation)
        if 'preservation' in opts.purpose:
            # Add derivation link and associated event
            insert_derivation_event(
                original_uuid=opts.file_uuid,
                output_uuid=manually_normalized_file.uuid,
                derivation_uuid=str(uuid.uuid4()),
                event_detail_output="manual normalization",
                outcome_detail_note=None,
            )
        return SUCCESS

    try:
        format_id = FileFormatVersion.objects.get(file_uuid=opts.file_uuid)
    # Can't do anything if the file wasn't identified
    except FileFormatVersion.DoesNotExist, FileFormatVersion. MultipleObjectsReturned:
        print('Not normalizing ',
            os.path.basename(file_.currentlocation),
            ' - file format not identified',
            file=sys.stderr)
        return NO_RULE_FOUND
    if format_id.format_version == None:
        print('Not normalizing',
            os.path.basename(file_.currentlocation),
            ' - file format not identified',
            file=sys.stderr)
        return NO_RULE_FOUND
    print('File format:', format_id.format_version)

    # Look up the normalization command in the FPR
    try:
        rule = FPRule.active.get(format=format_id.format_version,
        purpose=opts.purpose)
    except FPRule.DoesNotExist:
        try:
            rule = FPRule.active.get(purpose='default_'+opts.purpose)
            print("No rule for", os.path.basename(file_.currentlocation),
                "falling back to default", opts.purpose, "rule")
        except FPRule.DoesNotExist:
            print('Not normalizing', os.path.basename(file_.currentlocation),
                ' - No rule or default rule found to normalize for', opts.purpose,
                file=sys.stderr)
            return NO_RULE_FOUND
    print('Format Policy Rule:', rule)
    command = rule.command
    print('Format Policy Command', command.description)

    replacement_dict = get_replacement_dict(opts)
    cl = transcoder.CommandLinker(rule, command, replacement_dict, opts, once_normalized)
    exitstatus = cl.execute()

    # Store thumbnails locally for use during AIP searches
    # TODO is this still needed, with the storage service?
    if 'thumbnail' in opts.purpose:
        thumbnail_filepath = cl.commandObject.output_location
        clientConfigFilePath = '/etc/archivematica/MCPClient/clientConfig.conf'
        config = ConfigParser.SafeConfigParser()
        config.read(clientConfigFilePath)
        try:
            shared_path = config.get('MCPClient', 'sharedDirectoryMounted')
        except:
            shared_path = '/var/archivematica/sharedDirectory/'
        thumbnail_storage_dir = os.path.join(
            shared_path,
            'www',
            'thumbnails',
            opts.sip_uuid,
        )
        try:
            os.makedirs(thumbnail_storage_dir)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(thumbnail_storage_dir):
                pass
            else:
                raise
        thumbnail_basename, thumbnail_extension = os.path.splitext(thumbnail_filepath)
        thumbnail_storage_file = os.path.join(
            thumbnail_storage_dir,
            opts.file_uuid + thumbnail_extension,
        )

        shutil.copyfile(thumbnail_filepath, thumbnail_storage_file)

    if not exitstatus == 0:
        print('Command', command.description, 'failed!', file=sys.stderr)
        return RULE_FAILED
    else:
        print('Successfully normalized ', os.path.basename(opts.file_path), 'for', opts.purpose)
        return SUCCESS

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Identify file formats.')
    # sip dir
    parser.add_argument('purpose', type=str, help='"preservation", "access", "thumbnail"')
    parser.add_argument('file_uuid', type=str, help='%fileUUID%')
    parser.add_argument('file_path', type=str, help='%relativeLocation%')
    parser.add_argument('sip_path', type=str, help='%SIPDirectory%')
    parser.add_argument('sip_uuid', type=str, help='%SIPUUID%')
    parser.add_argument('task_uuid', type=str, help='%taskUUID%')
    parser.add_argument('normalize_file_grp_use', type=str, help='"service", "original", "submissionDocumentation", etc')

    opts = parser.parse_args()
    sys.exit(main(opts))

<commit_msg>normalize: replacement_dict should be ReplacementDict instance<commit_after>#!/usr/bin/python2 -OO
from __future__ import print_function
import argparse
import ConfigParser
import csv
import datetime
import errno
import os
import shutil
import sys
import traceback
import uuid

import transcoder

sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import databaseFunctions
import databaseInterface
import fileOperations
from dicts import ReplacementDict

path = '/usr/share/archivematica/dashboard'
if path not in sys.path:
    sys.path.append(path)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings.common'
from fpr.models import FPRule
from main.models import FileFormatVersion, File

# Return codes
SUCCESS = 0
RULE_FAILED = 1
NO_RULE_FOUND = 2

def get_replacement_dict(opts):
    """ Generates values for all knows %var% replacement variables. """
    prefix = ""
    postfix = ""
    output_dir = ""
    #get file name and extension
    (directory, basename) = os.path.split(opts.file_path)
    directory += os.path.sep  # All paths should have trailing /
    (filename, extension_dot) = os.path.splitext(basename)

    if "preservation" in opts.purpose:
        postfix = "-" + opts.task_uuid
        output_dir = directory
    elif "access" in opts.purpose:
        prefix = opts.file_uuid + "-"
        output_dir = os.path.join(opts.sip_path, "DIP", "objects") + os.path.sep
    elif "thumbnail" in opts.purpose:
        output_dir = os.path.join(opts.sip_path, "thumbnails") + os.path.sep
        postfix = opts.file_uuid
    else:
        print("Unsupported command purpose", opts.purpose, file=sys.stderr)
        return None

    # Populates the standard set of unit variables, so,
    # e.g., %fileUUID% is available
    replacement_dict = ReplacementDict.frommodel(type_='file',
                                                 file_=opts.file_uuid)

    output_filename = ''.join([prefix, filename, postfix])
    replacement_dict.update({
        "%outputDirectory%": output_dir,
        "%prefix%": prefix,
        "%postfix%": postfix,
        "%outputFileName%": output_filename, # does not include extension
        "%outputFilePath%": os.path.join(output_dir, output_filename) # does not include extension
    })
    return replacement_dict

def check_manual_normalization(opts):
    """ Checks for manually normalized file, returns that path or None.

    Checks by looking for access/preservation files for a give original file.

    Check the manualNormalization/access and manualNormalization/preservation
    directories for access and preservation files.  If a nomalization.csv
    file is specified, check there first for the mapping between original
    file and access/preservation file. """

    # If normalization.csv provided, check there for mapping from original
    # to access/preservation file
    normalization_csv = os.path.join(opts.sip_path, "objects", "manualNormalization", "normalization.csv")
    dirname, bname = os.path.split(opts.file_path)
    if os.path.isfile(normalization_csv):
        found = False
        with open(normalization_csv, 'rb') as csv_file:
            reader = csv.reader(csv_file)
            # Search the file for an original filename that matches the one provided
            try:
                for row in reader:
                    if "#" in row[0]: # ignore comments
                        continue
                    original, access_file, preservation_file = row
                    if original.lower() == bname.lower():
                        found = True
                        break
            except csv.Error:
                print("Error reading", normalization_csv, " on line", reader.line_num, file=sys.stderr)
                traceback.print_exc(file=sys.stderr)
                return None

        # If we didn't find a match, let it fall through to the usual method
        if found:
            # No manually normalized file for command classification
            if "preservation" in opts.purpose and not preservation_file:
                return None
            if "access" in opts.purpose and not access_file:
                return None

            # If we found a match, verify access/preservation exists in DB
            # match and pull original location b/c sanitization
            if "preservation" in opts.purpose:
                filename = preservation_file
            elif "access" in opts.purpose:
                filename = access_file
            else:
                return None
            print('Looking for', filename, 'in database')
            # FIXME: SQL uses removedtime=0. Convince Django to express this
            return File.objects.get(sip=opts.sip_uuid, originallocation__iendswith=filename) #removedtime = 0

    # Assume that any access/preservation file found with the right
    # name is the correct one
    bname, _ = os.path.splitext(bname)
    dirname = dirname.replace(opts.sip_path, '%SIPDirectory%')
    path = os.path.join(dirname, bname)
    if "preservation" in opts.purpose:
        path = path.replace("%SIPDirectory%objects/",
            "%SIPDirectory%objects/manualNormalization/preservation/")
    elif "access" in opts.purpose:
        path = path.replace("%SIPDirectory%objects/",
            "%SIPDirectory%objects/manualNormalization/access/")
    else:
        return None
    try:
        # FIXME: SQL uses removedtime=0. Cannot get Django to express this
        return File.objects.get(sip=opts.sip_uuid, currentlocation__startswith=path) #removedtime = 0
    except File.DoesNotExist, File.MultipleObjectsReturned:
        # No file with the correct path found, assume not manually normalized
        return None
    return None

def once_normalized(command, opts, replacement_dict):
    """ Updates the database if normalization completed successfully.

    Callback from transcoder.Command

    For preservation files, adds a normalization event, and derivation, as well
    as updating the size and checksum for the new file in the DB.  Adds format
    information for use in the METS file to FilesIDs.
    """
    transcoded_files = []
    if not command.output_location:
        command.output_location = ""
    if os.path.isfile(command.output_location):
        transcoded_files.append(command.output_location)
    elif os.path.isdir(command.output_location):
        for w in os.walk(command.output_location):
            path, _, files = w
            for p in files:
                p = os.path.join(path, p)
                if os.path.isfile(p):
                    transcoded_files.append(p)
    elif command.output_location:
        print("Error - output file does not exist [", command.output_location, "]", file=sys.stderr)
        command.exit_code = -2

    derivation_event_uuid = str(uuid.uuid4())
    event_detail_output = 'ArchivematicaFPRCommandID="{}"'.format(command.fpcommand.uuid)
    if command.event_detail_command is not None:
        event_detail_output += '; {}'.format(command.event_detail_command.std_out)
    for ef in transcoded_files:
        if "preservation" in opts.purpose:
            today = str(datetime.date.today())
            output_file_uuid = opts.task_uuid # File UUID is the same as task UUID for preservation
            # TODO Add manual normalization for files of same name mapping?
            #Add the new file to the SIP
            path_relative_to_sip = ef.replace(opts.sip_path, "%SIPDirectory%", 1)
            fileOperations.addFileToSIP(
                path_relative_to_sip,
                output_file_uuid, # File UUID
                opts.sip_uuid, # SIP UUID
                opts.task_uuid, # Task UUID
                today, # Current date
                sourceType="creation",
                use="preservation",
            )

            #Calculate new file checksum
            fileOperations.updateSizeAndChecksum(
                output_file_uuid, # File UUID, same as task UUID for preservation
                ef, # File path
                today, # Date
                str(uuid.uuid4()), # Event UUID, new UUID
            )

            # Add derivation link and associated event
            insert_derivation_event(
                original_uuid=opts.file_uuid,
                output_uuid=output_file_uuid,
                derivation_uuid=derivation_event_uuid,
                event_detail_output=event_detail_output,
                outcome_detail_note=path_relative_to_sip,
                today=today,
            )

            sql = "INSERT INTO FilesIDs (fileUUID, formatName, formatVersion, formatRegistryName, formatRegistryKey) VALUES ('%s', '%s', NULL, NULL, NULL);" % (output_file_uuid, command.fpcommand.output_format.description)
            databaseInterface.runSQL(sql)

def insert_derivation_event(original_uuid, output_uuid, derivation_uuid,
        event_detail_output, outcome_detail_note, today=None):
    """ Add the derivation link for preservation files and the event. """
    if today is None:
        today = str(datetime.date.today())
    # Add event information to current file
    databaseFunctions.insertIntoEvents(
       fileUUID=original_uuid,
       eventIdentifierUUID=derivation_uuid,
       eventType="normalization",
       eventDateTime=today,
       eventDetail=event_detail_output,
       eventOutcome="",
       eventOutcomeDetailNote=outcome_detail_note or "",
    )

    # Add linking information between files
    databaseFunctions.insertIntoDerivations(
        sourceFileUUID=original_uuid,
        derivedFileUUID=output_uuid,
        relatedEventUUID=derivation_uuid,
    )

def main(opts):
    """ Find and execute normalization commands on input file. """
    # TODO fix for maildir working only on attachments

    # Find the file and it's FormatVersion (file identification)
    try:
        file_ = File.objects.get(uuid=opts.file_uuid)
    except File.DoesNotExist:
        print('File with uuid', opts.file_uuid, 'does not exist in database.', file=sys.stderr)
        return NO_RULE_FOUND
    print('File found:', file_.uuid, file_.currentlocation)

    # Unless normalization file group use is submissionDocumentation, skip the
    # submissionDocumentation directory
    if opts.normalize_file_grp_use != "submissionDocumentation" and file_.currentlocation.startswith('%SIPDirectory%objects/submissionDocumentation'):
        print('File', os.path.basename(opts.file_path), 'in objects/submissionDocumentation, skipping')
        return SUCCESS

    # Only normalize files where the file's group use and normalize group use match
    if file_.filegrpuse != opts.normalize_file_grp_use:
        print(os.path.basename(opts.file_path), 'is file group usage', file_.filegrpuse, 'instead of ', opts.normalize_file_grp_use, ' - skipping')
        return SUCCESS

    # If a file has been manually normalized for this purpose, skip it
    manually_normalized_file = check_manual_normalization(opts)
    if manually_normalized_file:
        print(os.path.basename(opts.file_path), 'was already manually normalized into', manually_normalized_file.currentlocation)
        if 'preservation' in opts.purpose:
            # Add derivation link and associated event
            insert_derivation_event(
                original_uuid=opts.file_uuid,
                output_uuid=manually_normalized_file.uuid,
                derivation_uuid=str(uuid.uuid4()),
                event_detail_output="manual normalization",
                outcome_detail_note=None,
            )
        return SUCCESS

    try:
        format_id = FileFormatVersion.objects.get(file_uuid=opts.file_uuid)
    # Can't do anything if the file wasn't identified
    except FileFormatVersion.DoesNotExist, FileFormatVersion. MultipleObjectsReturned:
        print('Not normalizing ',
            os.path.basename(file_.currentlocation),
            ' - file format not identified',
            file=sys.stderr)
        return NO_RULE_FOUND
    if format_id.format_version == None:
        print('Not normalizing',
            os.path.basename(file_.currentlocation),
            ' - file format not identified',
            file=sys.stderr)
        return NO_RULE_FOUND
    print('File format:', format_id.format_version)

    # Look up the normalization command in the FPR
    try:
        rule = FPRule.active.get(format=format_id.format_version,
        purpose=opts.purpose)
    except FPRule.DoesNotExist:
        try:
            rule = FPRule.active.get(purpose='default_'+opts.purpose)
            print("No rule for", os.path.basename(file_.currentlocation),
                "falling back to default", opts.purpose, "rule")
        except FPRule.DoesNotExist:
            print('Not normalizing', os.path.basename(file_.currentlocation),
                ' - No rule or default rule found to normalize for', opts.purpose,
                file=sys.stderr)
            return NO_RULE_FOUND
    print('Format Policy Rule:', rule)
    command = rule.command
    print('Format Policy Command', command.description)

    replacement_dict = get_replacement_dict(opts)
    cl = transcoder.CommandLinker(rule, command, replacement_dict, opts, once_normalized)
    exitstatus = cl.execute()

    # Store thumbnails locally for use during AIP searches
    # TODO is this still needed, with the storage service?
    if 'thumbnail' in opts.purpose:
        thumbnail_filepath = cl.commandObject.output_location
        clientConfigFilePath = '/etc/archivematica/MCPClient/clientConfig.conf'
        config = ConfigParser.SafeConfigParser()
        config.read(clientConfigFilePath)
        try:
            shared_path = config.get('MCPClient', 'sharedDirectoryMounted')
        except:
            shared_path = '/var/archivematica/sharedDirectory/'
        thumbnail_storage_dir = os.path.join(
            shared_path,
            'www',
            'thumbnails',
            opts.sip_uuid,
        )
        try:
            os.makedirs(thumbnail_storage_dir)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(thumbnail_storage_dir):
                pass
            else:
                raise
        thumbnail_basename, thumbnail_extension = os.path.splitext(thumbnail_filepath)
        thumbnail_storage_file = os.path.join(
            thumbnail_storage_dir,
            opts.file_uuid + thumbnail_extension,
        )

        shutil.copyfile(thumbnail_filepath, thumbnail_storage_file)

    if not exitstatus == 0:
        print('Command', command.description, 'failed!', file=sys.stderr)
        return RULE_FAILED
    else:
        print('Successfully normalized ', os.path.basename(opts.file_path), 'for', opts.purpose)
        return SUCCESS

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Identify file formats.')
    # sip dir
    parser.add_argument('purpose', type=str, help='"preservation", "access", "thumbnail"')
    parser.add_argument('file_uuid', type=str, help='%fileUUID%')
    parser.add_argument('file_path', type=str, help='%relativeLocation%')
    parser.add_argument('sip_path', type=str, help='%SIPDirectory%')
    parser.add_argument('sip_uuid', type=str, help='%SIPUUID%')
    parser.add_argument('task_uuid', type=str, help='%taskUUID%')
    parser.add_argument('normalize_file_grp_use', type=str, help='"service", "original", "submissionDocumentation", etc')

    opts = parser.parse_args()
    sys.exit(main(opts))

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright (C) 2008 Samuel Abels, http://debain.org
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import gobject, gtk, cairo
from SpiffGtkWidgets import color
from TextBuffer      import TextBuffer
from Layout          import Layout
from AnnotationView  import AnnotationView

class TextEditor(gtk.TextView):
    def __init__(self, textbuffer = None, *args, **kwargs):
        if textbuffer is None:
            textbuffer = TextBuffer()
        gtk.TextView.__init__(self, textbuffer, *args, **kwargs)
        self.buffer           = textbuffer
        self.anno_width       = 200
        self.anno_padding     = 10
        self.anno_layout      = Layout(self)
        self.anno_views       = {}
        self.show_annotations = True
        self.exposed          = False
        self.set_right_margin(50 + self.anno_padding)
        self.connect('expose_event',        self._on_expose_event)
        self.connect('motion-notify-event', self._on_motion_notify_event)
        self.connect('event-after',         self._on_event_after)
        self.buffer.connect('mark-set',           self._on_buffer_mark_set)
        self.buffer.connect('annotation-added',   self._on_annotation_added)
        self.buffer.connect('annotation-removed', self._on_annotation_removed)
        self.set_wrap_mode(gtk.WRAP_WORD)

    def _on_motion_notify_event(self, editor, event):
        x, y = self.window_to_buffer_coords(gtk.TEXT_WINDOW_WIDGET,
                                            int(event.x),
                                            int(event.y))
        tags = self.get_iter_at_location(x, y).get_tags()

        # Without this call, further motion notify events don't get
        # triggered.
        self.window.get_pointer()

        # If any of the tags are links, show a hand.
        cursor = gtk.gdk.Cursor(gtk.gdk.XTERM)
        for tag in tags:
            if tag.get_data('link'):
                cursor = gtk.gdk.Cursor(gtk.gdk.HAND2)
                break

        self.get_window(gtk.TEXT_WINDOW_TEXT).set_cursor(cursor)
        return False

    def _on_event_after(self, textview, event):
        # Handle links here. Only when a button was released.
        if event.type != gtk.gdk.BUTTON_RELEASE:
            return False
        if event.button != 1:
            return False

        # Don't follow a link if the user has selected something.
        bounds = self.buffer.get_selection_bounds()
        if bounds:
            start, end = bounds
            if start.get_offset() != end.get_offset():
                return False

        # Check whether the cursor is pointing at a link.
        x, y = self.window_to_buffer_coords(gtk.TEXT_WINDOW_WIDGET,
                                            int(event.x),
                                            int(event.y))
        iter = textview.get_iter_at_location(x, y)

        for tag in iter.get_tags():
            link = tag.get_data('link')
            if not link:
                continue
            self.emit('link-clicked', link)
            break
        return False

    def _on_buffer_mark_set(self, buffer, iter, mark):
        self._update_annotations()

    def _on_expose_event(self, widget, event):
        if not self.exposed:
            self.exposed = True
            self._update_annotation_area()
            self._update_annotations()

        text_window = widget.get_window(gtk.TEXT_WINDOW_TEXT)
        if event.window != text_window:
            return

        # Create the cairo context.
        ctx = event.window.cairo_create()

        # Restrict Cairo to the exposed area; avoid extra work
        ctx.rectangle(event.area.x,
                      event.area.y,
                      event.area.width,
                      event.area.height)
        ctx.clip()

        self.draw(ctx, *event.window.get_size())

    def draw(self, ctx, w, h):
        if ctx is None:
            return
        if not self.show_annotations:
            return

        # Draw the dashes that connect annotations to their marker.
        ctx.set_line_width(1)
        ctx.set_dash((3, 2))
        right_margin = self.get_right_margin()
        for annotation in self.anno_layout.get_children():
            mark_x, mark_y = self._get_annotation_mark_position(annotation)
            anno_x, anno_y, anno_w, anno_h, d = annotation.window.get_geometry()
            path = [(mark_x,           mark_y - 5),
                    (mark_x,           mark_y),
                    (w - right_margin, mark_y),
                    (w,                anno_y + anno_h / 2)]

            stroke_color = annotation.get_border_color()
            ctx.set_source_rgba(*color.to_rgba(stroke_color))
            ctx.move_to(*path[0])
            for x, y in path[1:]:
                ctx.line_to(x, y)
            ctx.stroke()

    def _get_annotation_mark_offset(self, view1, view2):
        mark1 = view1.annotation.start_mark
        mark2 = view2.annotation.start_mark
        iter1 = self.get_buffer().get_iter_at_mark(mark1)
        iter2 = self.get_buffer().get_iter_at_mark(mark2)
        rect1 = self.get_iter_location(iter1)
        rect2 = self.get_iter_location(iter2)
        if rect1.y != rect2.y:
            return rect1.y - rect2.y
        return rect2.x - rect1.x

    def _get_annotation_mark_position(self, view):
        start_mark     = view.annotation.start_mark
        iter           = self.get_buffer().get_iter_at_mark(start_mark)
        rect           = self.get_iter_location(iter)
        mark_x, mark_y = rect.x, rect.y + rect.height
        return self.buffer_to_window_coords(gtk.TEXT_WINDOW_TEXT,
                                            mark_x, mark_y)

    def _update_annotation(self, annotation):
        # Resize the annotation.
        item_width = self.anno_width - 2 * self.anno_padding
        annotation.set_size_request(item_width, -1)

        # Find the x, y of the annotation's mark.
        mark_x, mark_y = self._get_annotation_mark_position(annotation)
        self.anno_layout.pull(annotation, mark_y)

    def _update_annotation_area(self):
        # Update the width and color of the annotation area.
        self.set_border_window_size(gtk.TEXT_WINDOW_RIGHT, self.anno_width)
        bg_color = self.get_style().base[gtk.STATE_NORMAL]
        window   = self.get_window(gtk.TEXT_WINDOW_RIGHT)
        if window:
            window.set_background(bg_color)

    def _update_annotations(self):
        if not self.show_annotations:
            return

        # Sort the annotations by line/char number and update them.
        iter   = self.get_buffer().get_end_iter()
        rect   = self.get_iter_location(iter)
        height = rect.y + rect.height
        self.anno_layout.sort(self._get_annotation_mark_offset)
        for annotation in self.anno_layout.get_children():
            self._update_annotation(annotation)
        self.anno_layout.update(self.anno_width, height)

        # Update lines.
        self.queue_draw()

    def set_annotation_area_width(self, width, padding = 10):
        self.anno_width   = width
        self.anno_padding = padding
        self._update_annotations()

    def _on_annotation_added(self, buffer, annotation):
        if self.show_annotations == False:
            return
        annotation.set_display_buffer(self.buffer)
        view = AnnotationView(annotation)
        self.anno_views[annotation] = view
        for event in ('focus-in-event', 'focus-out-event'):
            view.connect(event, self._on_annotation_event, annotation, event)
        view.show_all()
        self._update_annotation_area()
        self.anno_layout.add(view)
        self.add_child_in_window(view,
                                 gtk.TEXT_WINDOW_RIGHT,
                                 self.anno_padding,
                                 0)
        self._update_annotations()

    def _on_annotation_removed(self, buffer, annotation):
        view = self.anno_views[annotation]
        self.anno_layout.remove(view)
        self.remove(view)

    def _on_annotation_event(self, buffer, *args):
        annotation = args[-2]
        event_name = args[-1]
        self.emit('annotation-' + event_name, annotation)

    def set_show_annotations(self, active = True):
        if self.show_annotations == active:
            return

        # Unfortunately gtk.TextView deletes all children from the
        # border window if its size is 0. So we must re-add them when the
        # window reappears.
        self.show_annotations = active
        if active:
            for annotation in self.buffer.get_annotations():
                self._on_annotation_added(self.buffer, annotation)
        else:
            for annotation in self.buffer.get_annotations():
                self._on_annotation_removed(self.buffer, annotation)
            self.set_border_window_size(gtk.TEXT_WINDOW_RIGHT, 0)

gobject.signal_new('link-clicked',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))

gobject.signal_new('annotation-focus-in-event',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))

gobject.signal_new('annotation-focus-out-event',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))
<commit_msg>Make the link feature of the textview configurable.<commit_after># -*- coding: UTF-8 -*-
# Copyright (C) 2008 Samuel Abels, http://debain.org
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import gobject, gtk, cairo
from SpiffGtkWidgets import color
from TextBuffer      import TextBuffer
from Layout          import Layout
from AnnotationView  import AnnotationView

class TextEditor(gtk.TextView):
    def __init__(self, textbuffer = None, *args, **kwargs):
        if textbuffer is None:
            textbuffer = TextBuffer()
        gtk.TextView.__init__(self, textbuffer, *args, **kwargs)
        self.buffer           = textbuffer
        self.anno_width       = 200
        self.anno_padding     = 10
        self.anno_layout      = Layout(self)
        self.anno_views       = {}
        self.show_annotations = True
        self.handle_links     = True
        self.exposed          = False
        self.set_right_margin(50 + self.anno_padding)
        self.connect('expose_event',        self._on_expose_event)
        self.connect('motion-notify-event', self._on_motion_notify_event)
        self.connect('event-after',         self._on_event_after)
        self.buffer.connect('mark-set',           self._on_buffer_mark_set)
        self.buffer.connect('annotation-added',   self._on_annotation_added)
        self.buffer.connect('annotation-removed', self._on_annotation_removed)
        self.set_wrap_mode(gtk.WRAP_WORD)

    def _on_motion_notify_event(self, editor, event):
        if not self.handle_links:
            return
        x, y = self.window_to_buffer_coords(gtk.TEXT_WINDOW_WIDGET,
                                            int(event.x),
                                            int(event.y))
        tags = self.get_iter_at_location(x, y).get_tags()

        # Without this call, further motion notify events don't get
        # triggered.
        self.window.get_pointer()

        # If any of the tags are links, show a hand.
        cursor = gtk.gdk.Cursor(gtk.gdk.XTERM)
        for tag in tags:
            if tag.get_data('link'):
                cursor = gtk.gdk.Cursor(gtk.gdk.HAND2)
                break

        self.get_window(gtk.TEXT_WINDOW_TEXT).set_cursor(cursor)
        return False

    def _on_event_after(self, textview, event):
        # Handle links here. Only when a button was released.
        if event.type != gtk.gdk.BUTTON_RELEASE:
            return False
        if event.button != 1:
            return False
        if not self.handle_links:
            return

        # Don't follow a link if the user has selected something.
        bounds = self.buffer.get_selection_bounds()
        if bounds:
            start, end = bounds
            if start.get_offset() != end.get_offset():
                return False

        # Check whether the cursor is pointing at a link.
        x, y = self.window_to_buffer_coords(gtk.TEXT_WINDOW_WIDGET,
                                            int(event.x),
                                            int(event.y))
        iter = textview.get_iter_at_location(x, y)

        for tag in iter.get_tags():
            link = tag.get_data('link')
            if not link:
                continue
            self.emit('link-clicked', link)
            break
        return False

    def _on_buffer_mark_set(self, buffer, iter, mark):
        self._update_annotations()

    def _on_expose_event(self, widget, event):
        if not self.exposed:
            self.exposed = True
            self._update_annotation_area()
            self._update_annotations()

        text_window = widget.get_window(gtk.TEXT_WINDOW_TEXT)
        if event.window != text_window:
            return

        # Create the cairo context.
        ctx = event.window.cairo_create()

        # Restrict Cairo to the exposed area; avoid extra work
        ctx.rectangle(event.area.x,
                      event.area.y,
                      event.area.width,
                      event.area.height)
        ctx.clip()

        self.draw(ctx, *event.window.get_size())

    def draw(self, ctx, w, h):
        if ctx is None:
            return
        if not self.show_annotations:
            return

        # Draw the dashes that connect annotations to their marker.
        ctx.set_line_width(1)
        ctx.set_dash((3, 2))
        right_margin = self.get_right_margin()
        for annotation in self.anno_layout.get_children():
            mark_x, mark_y = self._get_annotation_mark_position(annotation)
            anno_x, anno_y, anno_w, anno_h, d = annotation.window.get_geometry()
            path = [(mark_x,           mark_y - 5),
                    (mark_x,           mark_y),
                    (w - right_margin, mark_y),
                    (w,                anno_y + anno_h / 2)]

            stroke_color = annotation.get_border_color()
            ctx.set_source_rgba(*color.to_rgba(stroke_color))
            ctx.move_to(*path[0])
            for x, y in path[1:]:
                ctx.line_to(x, y)
            ctx.stroke()

    def _get_annotation_mark_offset(self, view1, view2):
        mark1 = view1.annotation.start_mark
        mark2 = view2.annotation.start_mark
        iter1 = self.get_buffer().get_iter_at_mark(mark1)
        iter2 = self.get_buffer().get_iter_at_mark(mark2)
        rect1 = self.get_iter_location(iter1)
        rect2 = self.get_iter_location(iter2)
        if rect1.y != rect2.y:
            return rect1.y - rect2.y
        return rect2.x - rect1.x

    def _get_annotation_mark_position(self, view):
        start_mark     = view.annotation.start_mark
        iter           = self.get_buffer().get_iter_at_mark(start_mark)
        rect           = self.get_iter_location(iter)
        mark_x, mark_y = rect.x, rect.y + rect.height
        return self.buffer_to_window_coords(gtk.TEXT_WINDOW_TEXT,
                                            mark_x, mark_y)

    def _update_annotation(self, annotation):
        # Resize the annotation.
        item_width = self.anno_width - 2 * self.anno_padding
        annotation.set_size_request(item_width, -1)

        # Find the x, y of the annotation's mark.
        mark_x, mark_y = self._get_annotation_mark_position(annotation)
        self.anno_layout.pull(annotation, mark_y)

    def _update_annotation_area(self):
        # Update the width and color of the annotation area.
        self.set_border_window_size(gtk.TEXT_WINDOW_RIGHT, self.anno_width)
        bg_color = self.get_style().base[gtk.STATE_NORMAL]
        window   = self.get_window(gtk.TEXT_WINDOW_RIGHT)
        if window:
            window.set_background(bg_color)

    def _update_annotations(self):
        if not self.show_annotations:
            return

        # Sort the annotations by line/char number and update them.
        iter   = self.get_buffer().get_end_iter()
        rect   = self.get_iter_location(iter)
        height = rect.y + rect.height
        self.anno_layout.sort(self._get_annotation_mark_offset)
        for annotation in self.anno_layout.get_children():
            self._update_annotation(annotation)
        self.anno_layout.update(self.anno_width, height)

        # Update lines.
        self.queue_draw()

    def set_annotation_area_width(self, width, padding = 10):
        self.anno_width   = width
        self.anno_padding = padding
        self._update_annotations()

    def _on_annotation_added(self, buffer, annotation):
        if self.show_annotations == False:
            return
        annotation.set_display_buffer(self.buffer)
        view = AnnotationView(annotation)
        self.anno_views[annotation] = view
        for event in ('focus-in-event', 'focus-out-event'):
            view.connect(event, self._on_annotation_event, annotation, event)
        view.show_all()
        self._update_annotation_area()
        self.anno_layout.add(view)
        self.add_child_in_window(view,
                                 gtk.TEXT_WINDOW_RIGHT,
                                 self.anno_padding,
                                 0)
        self._update_annotations()

    def _on_annotation_removed(self, buffer, annotation):
        view = self.anno_views[annotation]
        self.anno_layout.remove(view)
        self.remove(view)

    def _on_annotation_event(self, buffer, *args):
        annotation = args[-2]
        event_name = args[-1]
        self.emit('annotation-' + event_name, annotation)

    def set_show_annotations(self, active = True):
        if self.show_annotations == active:
            return

        # Unfortunately gtk.TextView deletes all children from the
        # border window if its size is 0. So we must re-add them when the
        # window reappears.
        self.show_annotations = active
        if active:
            for annotation in self.buffer.get_annotations():
                self._on_annotation_added(self.buffer, annotation)
        else:
            for annotation in self.buffer.get_annotations():
                self._on_annotation_removed(self.buffer, annotation)
            self.set_border_window_size(gtk.TEXT_WINDOW_RIGHT, 0)

    def set_handle_links(self, handle):
        """
        Defines whether the cursor is changed and whether clicks are accepted
        when hovering over text with tags that have data named "link"
        attached.
        """
        self.handle_links = handle

gobject.signal_new('link-clicked',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))

gobject.signal_new('annotation-focus-in-event',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))

gobject.signal_new('annotation-focus-out-event',
                   TextEditor,
                   gobject.SIGNAL_RUN_FIRST,
                   gobject.TYPE_NONE,
                   (gobject.TYPE_PYOBJECT, ))
<|endoftext|>


<commit_before>import contextlib

from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import profiler  # NOQA
from cupy.cuda import runtime  # NOQA
from cupy.cuda import stream  # NOQA

_available = None

if runtime.runtimeGetVersion() >= 8000:
    from cupy.cuda import cusolver  # NOQA
    cusolver_enabled = True
else:
    cusolver_enabled = False

try:
    from cupy.cuda import nvtx  # NOQA
    nvtx_enabled = True
except ImportError:
    nvtx_enabled = False

try:
    from cupy.cuda import thrust  # NOQA
    thrust_enabled = True
except ImportError:
    thrust_enabled = False

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
    return _available

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<commit_msg>Use build version<commit_after>import contextlib

from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import driver  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import profiler  # NOQA
from cupy.cuda import runtime  # NOQA
from cupy.cuda import stream  # NOQA

_available = None

if driver.get_build_version() >= 8000:
    from cupy.cuda import cusolver  # NOQA
    cusolver_enabled = True
else:
    cusolver_enabled = False

try:
    from cupy.cuda import nvtx  # NOQA
    nvtx_enabled = True
except ImportError:
    nvtx_enabled = False

try:
    from cupy.cuda import thrust  # NOQA
    thrust_enabled = True
except ImportError:
    thrust_enabled = False

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
    return _available

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<|endoftext|>


<commit_before>
#combinaciones

import itertools

dictt={'a':[1,2,3],'b':['g', 1], 'c':[6]}

varNames = sorted(dictt)
combinations = [dict(zip(varNames, prod)) for prod in itertools.product(*(dictt[varName] for varName in varNames))]

for i in combinations:
    print(i)
#print(combinations)
<commit_msg>pruebas<commit_after>
#combinaciones

import itertools

parametros = {
                'lr_pesos':  [0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15],
                'unidad':    ['binaria', 'gaussiana'],
                'tamBatch':  [10],
                'epocasTRN': [100,200,300],
                'epocasFIT': [2000],

            }

varNames = sorted(parametros)
combinations = [dict(zip(varNames, prod)) for prod in itertools.product(*(parametros[varName] for varName in varNames))]

print(type(combinations))
print(type(combinations[0]))
for i in combinations:
    print(i)

print("Cantidad de Combinaciones posibles: ", len(combinations))
tiempo_x_ejecucion = 6
cantidad_dias = len(combinations) * tiempo_x_ejecucion / 24.0
print("Cantidad de dias continuos de ejecucion: ", cantidad_dias)
#print(combinations)
<|endoftext|>


<commit_before># -*- encoding: UTF-8 -*-

from cvn import settings as stCVN
import datetime

def parse_scope(treeXML):
    '''Input: Scope node
       Example: CvnItem/Link/Scope'''
    dataCVN = {}
    if treeXML:
        dataCVN['ambito'] = unicode(stCVN.SCOPE[treeXML.find(
            'Type/Item').text.strip()])
        if dataCVN['ambito'] == u'Otros' and treeXML.find('Others/Item'):
            dataCVN['otro_ambito'] = unicode(treeXML.find(
                'Others/Item').text.strip())
    return dataCVN

def _parse_duration(duration):
    '''Input: Duration/Item node
       Example: CvnItem/Date/Duration/Item
    '''
    duration = duration.text
    duracion = 0
    number = ''
    for item in duration[1:]:
        if item.isdigit():
            number = item
        else:
            if item == 'Y':
                duracion += int(unicode(number)) * 365
            if item == 'M':
                duracion += int(unicode(number)) * 30
            if item == 'D':
                duracion += int(unicode(number))
    return duracion

def parse_nif(xml):
    '''Input: root node'''
    if xml is None:
        return ''
    nif = ''
    id_node = xml.find(
        'Agent/Identification/PersonalIdentification/OfficialId')
    if id_node is None:
        return nif
    nif_node = id_node.find('DNI/Item')
    if nif_node is None:
        nif_node = id_node.find('NIE/Item')

    if nif_node is not None:
        nif = nif_node.text.strip()

    return nif

def parse_authors(author_list):
    '''Input: A list of Author nodes'''
    authors = ''
    for author in author_list:
        authorItem = ''
        if (author.find("GivenName/Item") and
           author.find("GivenName/Item").text):
            authorItem = unicode(author.find(
                "GivenName/Item").text.strip())
        if (author.find("FirstFamilyName/Item") and
           author.find("FirstFamilyName/Item").text):
            authorItem += u' ' + unicode(author.find(
                "FirstFamilyName/Item").text.strip())
        if (author.find("SecondFamilyName/Item") and
           author.find("SecondFamilyName/Item").text):
            authorItem += u' ' + unicode(
                author.find("SecondFamilyName/Item").text.strip())
        if author.find("Signature/Item").text:
            if authorItem:
                authors += unicode(
                    authorItem + ' (' + author.find(
                        "Signature/Item").text.strip() + '); ')
            else:
                authors += unicode(author.find(
                    "Signature/Item").text.strip() + '; ')
        else:
            authors += authorItem + '; '
    return authors[:-2]

def parse_produccion_type(xml):
    '''Input: CvnItem node'''
    if xml is None:
        return ''
    cvn_key = xml.find('CvnItemID/CVNPK/Item').text.strip()
    if cvn_key not in stCVN.FECYT_CODE:
        return ''
    return stCVN.FECYT_CODE[cvn_key]

def parse_produccion_subtype(xml):
    '''Input: CvnItem node'''
    if xml is None:
        return ''
    subtype = xml.find('Subtype/SubType1/Item')
    if subtype is None:
        return ''
    subtype = subtype.text.strip()
    if subtype in stCVN.FECYT_CODE_SUBTYPE:
        return stCVN.FECYT_CODE_SUBTYPE[subtype]
    return ''

def parse_publicacion_location(treeXML):
    '''Input: Location node'''
    data = {}
    if treeXML:
        volume = treeXML.find('Volume/Item')
        if volume is not None and volume.text is not None:
            data['volumen'] = volume.text.strip()
        number = treeXML.find('Number/Item')
        if number is not None and number.text is not None:
            data['numero'] = number.text.strip()
        page = treeXML.find('InitialPage/Item')
        if page is not None and page.text is not None:
            data['pagina_inicial'] = page.text.strip()
        page = treeXML.find('FinalPage/Item')
        if page is not None and page.text is not None:
            data['pagina_final'] = page.text.strip()
    return data

def _parse_segregated_date(xml):
    date = xml.find('DayMonthYear/Item')
    if date is not None:
        date = date.text.split('-')
        return datetime.date(int(date[0]), int(date[1]), int(date[2]))
    date = xml.find('MonthYear/Item')
    if date is not None:
        date = date.text.split('-')
        return datetime.date(int(date[0]), int(date[1]), 1)
    date = xml.find('Year/Item')
    if date is not None:
        return datetime.date(int(date.text), 1, 1)
    return None

def _parse_unitary_date(xml):
    date = xml.text.split('-')
    return datetime.date(int(date[0]), int(date[1]), int(date[2]))

def parse_date(xml):
    '''Input: date node'''
    if xml is None:
        return None
    # Node of type Date > OnlyDate
    node = xml.find('OnlyDate')
    if node is not None:
        return _parse_segregated_date(node)
    # Node of type Date > StartDate
    node = xml.find('StartDate')
    if node is not None:
        return _parse_segregated_date(node)
    # Node of type Date > Item
    node = xml.find('Item')
    if node is not None:
        return _parse_unitary_date(node)
    return None

def parse_date_interval(xml):
    '''Input: date node'''
    if xml is None:
        return None, None, None
    # Get start date
    fecha_inicio = parse_date(xml)
    fecha_fin_1 = None
    fecha_fin_2 = None
    duration_1 = 0
    duration_2 = 0
    # Get end date
    node = xml.find('EndDate')
    if node is not None:
        fecha_fin_1 = _parse_segregated_date(node)
        '''If fecha_inicio is None duration is useless. So in the case
           of being a fecha_fin with no fecha inicio we just return it.'''
        if fecha_inicio is None:
            return None, fecha_fin_1, None
        delta = fecha_fin_1 - fecha_inicio
        duration_1 = delta.days
    # Get duration
    node = xml.find('Duration/Item')
    if node is not None:
        duration_2 = _parse_duration(node)
        if fecha_inicio is None:
            return None, None, None
        fecha_fin_2 = fecha_inicio + datetime.timedelta(days=duration_2)
    # Chose what end date to return. The first if is needed so fecha_fin
    # is None instead of equal to fecha_inicio in certain conditions.
    if fecha_fin_1 is None and fecha_fin_2 is None:
        return fecha_inicio, None, None
    if duration_1 > duration_2:
        return fecha_inicio, fecha_fin_1, duration_1
    else:
        return fecha_inicio, fecha_fin_2, duration_2

def parse_produccion_id(id_list, code_type):
    '''Input: id_list is a list of ExternalPK nodes (ExternalPK nodes contain
       different identifiers for produccion nodes. code_type is what
       type of id you want to get: ISSN, ISBN, etc. (PRODUCCION_ID_CODE)'''
    for node in id_list:
        if node.find('Type/Item').text != code_type:
            continue
        return node.find('Code/Item').text
    return None
<commit_msg>Put private functions at start of file<commit_after># -*- encoding: UTF-8 -*-

from cvn import settings as stCVN
import datetime

def _parse_duration(duration):
    '''Input: Duration/Item node
       Example: CvnItem/Date/Duration/Item
    '''
    duration = duration.text
    duracion = 0
    number = ''
    for item in duration[1:]:
        if item.isdigit():
            number = item
        else:
            if item == 'Y':
                duracion += int(unicode(number)) * 365
            if item == 'M':
                duracion += int(unicode(number)) * 30
            if item == 'D':
                duracion += int(unicode(number))
    return duracion

def _parse_segregated_date(xml):
    date = xml.find('DayMonthYear/Item')
    if date is not None:
        date = date.text.split('-')
        return datetime.date(int(date[0]), int(date[1]), int(date[2]))
    date = xml.find('MonthYear/Item')
    if date is not None:
        date = date.text.split('-')
        return datetime.date(int(date[0]), int(date[1]), 1)
    date = xml.find('Year/Item')
    if date is not None:
        return datetime.date(int(date.text), 1, 1)
    return None

def _parse_unitary_date(xml):
    date = xml.text.split('-')
    return datetime.date(int(date[0]), int(date[1]), int(date[2]))

def parse_scope(treeXML):
    '''Input: Scope node
       Example: CvnItem/Link/Scope'''
    dataCVN = {}
    if treeXML:
        dataCVN['ambito'] = unicode(stCVN.SCOPE[treeXML.find(
            'Type/Item').text.strip()])
        if dataCVN['ambito'] == u'Otros' and treeXML.find('Others/Item'):
            dataCVN['otro_ambito'] = unicode(treeXML.find(
                'Others/Item').text.strip())
    return dataCVN

def parse_nif(xml):
    '''Input: root node'''
    if xml is None:
        return ''
    nif = ''
    id_node = xml.find(
        'Agent/Identification/PersonalIdentification/OfficialId')
    if id_node is None:
        return nif
    nif_node = id_node.find('DNI/Item')
    if nif_node is None:
        nif_node = id_node.find('NIE/Item')

    if nif_node is not None:
        nif = nif_node.text.strip()

    return nif

def parse_authors(author_list):
    '''Input: A list of Author nodes'''
    authors = ''
    for author in author_list:
        authorItem = ''
        if (author.find("GivenName/Item") and
           author.find("GivenName/Item").text):
            authorItem = unicode(author.find(
                "GivenName/Item").text.strip())
        if (author.find("FirstFamilyName/Item") and
           author.find("FirstFamilyName/Item").text):
            authorItem += u' ' + unicode(author.find(
                "FirstFamilyName/Item").text.strip())
        if (author.find("SecondFamilyName/Item") and
           author.find("SecondFamilyName/Item").text):
            authorItem += u' ' + unicode(
                author.find("SecondFamilyName/Item").text.strip())
        if author.find("Signature/Item").text:
            if authorItem:
                authors += unicode(
                    authorItem + ' (' + author.find(
                        "Signature/Item").text.strip() + '); ')
            else:
                authors += unicode(author.find(
                    "Signature/Item").text.strip() + '; ')
        else:
            authors += authorItem + '; '
    return authors[:-2]

def parse_produccion_type(xml):
    '''Input: CvnItem node'''
    if xml is None:
        return ''
    cvn_key = xml.find('CvnItemID/CVNPK/Item').text.strip()
    if cvn_key not in stCVN.FECYT_CODE:
        return ''
    return stCVN.FECYT_CODE[cvn_key]

def parse_produccion_subtype(xml):
    '''Input: CvnItem node'''
    if xml is None:
        return ''
    subtype = xml.find('Subtype/SubType1/Item')
    if subtype is None:
        return ''
    subtype = subtype.text.strip()
    if subtype in stCVN.FECYT_CODE_SUBTYPE:
        return stCVN.FECYT_CODE_SUBTYPE[subtype]
    return ''

def parse_publicacion_location(treeXML):
    '''Input: Location node'''
    data = {}
    if treeXML:
        volume = treeXML.find('Volume/Item')
        if volume is not None and volume.text is not None:
            data['volumen'] = volume.text.strip()
        number = treeXML.find('Number/Item')
        if number is not None and number.text is not None:
            data['numero'] = number.text.strip()
        page = treeXML.find('InitialPage/Item')
        if page is not None and page.text is not None:
            data['pagina_inicial'] = page.text.strip()
        page = treeXML.find('FinalPage/Item')
        if page is not None and page.text is not None:
            data['pagina_final'] = page.text.strip()
    return data

def parse_date(xml):
    '''Input: date node'''
    if xml is None:
        return None
    # Node of type Date > OnlyDate
    node = xml.find('OnlyDate')
    if node is not None:
        return _parse_segregated_date(node)
    # Node of type Date > StartDate
    node = xml.find('StartDate')
    if node is not None:
        return _parse_segregated_date(node)
    # Node of type Date > Item
    node = xml.find('Item')
    if node is not None:
        return _parse_unitary_date(node)
    return None

def parse_date_interval(xml):
    '''Input: date node'''
    if xml is None:
        return None, None, None
    # Get start date
    fecha_inicio = parse_date(xml)
    fecha_fin_1 = None
    fecha_fin_2 = None
    duration_1 = 0
    duration_2 = 0
    # Get end date
    node = xml.find('EndDate')
    if node is not None:
        fecha_fin_1 = _parse_segregated_date(node)
        '''If fecha_inicio is None duration is useless. So in the case
           of being a fecha_fin with no fecha inicio we just return it.'''
        if fecha_inicio is None:
            return None, fecha_fin_1, None
        delta = fecha_fin_1 - fecha_inicio
        duration_1 = delta.days
    # Get duration
    node = xml.find('Duration/Item')
    if node is not None:
        duration_2 = _parse_duration(node)
        if fecha_inicio is None:
            return None, None, None
        fecha_fin_2 = fecha_inicio + datetime.timedelta(days=duration_2)
    # Chose what end date to return. The first if is needed so fecha_fin
    # is None instead of equal to fecha_inicio in certain conditions.
    if fecha_fin_1 is None and fecha_fin_2 is None:
        return fecha_inicio, None, None
    if duration_1 > duration_2:
        return fecha_inicio, fecha_fin_1, duration_1
    else:
        return fecha_inicio, fecha_fin_2, duration_2

def parse_produccion_id(id_list, code_type):
    '''Input: id_list is a list of ExternalPK nodes (ExternalPK nodes contain
       different identifiers for produccion nodes. code_type is what
       type of id you want to get: ISSN, ISBN, etc. (PRODUCCION_ID_CODE)'''
    for node in id_list:
        if node.find('Type/Item').text != code_type:
            continue
        return node.find('Code/Item').text
    return None
<|endoftext|>


<commit_before># Copyright (c) 2012, Calxeda Inc.
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Calxeda Inc. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.

from collections import deque
from threading import Thread, Lock
from time import sleep

class Task(object):
    """ A task object representing some unit of work to be done. """
    def __init__(self, method, *args):
        self._method = method
        self._args = args

        self.status = "Queued"

    def join(self):
        """ Wait for this task to finish """
        while self.is_alive():
            pass # TODO: don't busy wait here

    def is_alive(self):
        """ Return true if this task hasn't been finished """
        return not self.status in ["Completed", "Failed"]

class TaskQueue(object):
    """ A task queue, consisting of a queue and a number of workers. """

    def __init__(self, threads=48, delay=0):
        self.threads = threads
        self.delay = delay

        self._lock = Lock()
        self._queue = deque()
        self._workers = 0

    def put(self, method, *args):
        """
        Add a task to the task queue, and spawn a worker if we're not full.

        method: a method to call
        args: args to pass to the method

        returns: a Task object that will be executed by a worker at a later
        time.
        """
        self._lock.acquire()

        task = Task(method, *args)
        self._queue.append(task)

        if self._workers < self.threads:
            TaskWorker(task_queue=self, delay=self.delay)
            self._workers += 1

        self._lock.release()
        return task

    def get(self):
        """
        Get a task from the task queue. Mainly used by workers.

        returns: a Task object that hasn't been executed yet.
        raises: IndexError if there are no tasks in the queue.
        """
        self._lock.acquire()
        try:
            return self._queue.popleft()
        finally:
            self._lock.release()

    def _remove_worker(self):
        """ Decrement the worker count. Should only be used by TaskWorker. """
        self._lock.acquire()
        self._workers -= 1
        self._lock.release()

class TaskWorker(Thread):
    """ A worker thread that runs tasks from a TaskQueue. """
    def __init__(self, task_queue, delay=0):
        super(TaskWorker, self).__init__()
        self.daemon = True

        self._task_queue = task_queue
        self._delay = delay

        self.start()

    def run(self):
        """ Repeatedly get tasks from the TaskQueue and execute them. """
        try:
            while True:
                sleep(self._delay)
                task = self._task_queue.get()
                task.status = "In Progress"
                try:
                    task.result = task._method(*task._args)
                    task.status = "Completed"
                except Exception as e:
                    task.error = e
                    task.status = "Failed"
        except:
            self._task_queue._remove_worker()

DEFAULT_TASK_QUEUE = TaskQueue()
<commit_msg>CXMAN-150: Don't busy-wait on Task.join()<commit_after># Copyright (c) 2012, Calxeda Inc.
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Calxeda Inc. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.

from collections import deque
from threading import Thread, Lock, Event
from time import sleep

class Task(object):
    """ A task object representing some unit of work to be done. """
    def __init__(self, method, *args):
        self.status = "Queued"

        self._method = method
        self._args = args
        self._finished = Event()

    def join(self):
        """ Wait for this task to finish """
        self._finished.wait()

    def is_alive(self):
        """ Return true if this task hasn't been finished """
        return not self._finished.is_set()

    def _run(self):
        """ Execute this task. Should only be called by TaskWorker. """
        self.status = "In Progress"
        try:
            self.result = self._method(*self._args)
            self.status = "Completed"
        except Exception as e:
            self.error = e
            self.status = "Failed"

        self._finished.set()

class TaskQueue(object):
    """ A task queue, consisting of a queue and a number of workers. """

    def __init__(self, threads=48, delay=0):
        self.threads = threads
        self.delay = delay

        self._lock = Lock()
        self._queue = deque()
        self._workers = 0

    def put(self, method, *args):
        """
        Add a task to the task queue, and spawn a worker if we're not full.

        method: a method to call
        args: args to pass to the method

        returns: a Task object that will be executed by a worker at a later
        time.
        """
        self._lock.acquire()

        task = Task(method, *args)
        self._queue.append(task)

        if self._workers < self.threads:
            TaskWorker(task_queue=self, delay=self.delay)
            self._workers += 1

        self._lock.release()
        return task

    def get(self):
        """
        Get a task from the task queue. Mainly used by workers.

        returns: a Task object that hasn't been executed yet.
        raises: IndexError if there are no tasks in the queue.
        """
        self._lock.acquire()
        try:
            return self._queue.popleft()
        finally:
            self._lock.release()

    def _remove_worker(self):
        """ Decrement the worker count. Should only be used by TaskWorker. """
        self._lock.acquire()
        self._workers -= 1
        self._lock.release()

class TaskWorker(Thread):
    """ A worker thread that runs tasks from a TaskQueue. """
    def __init__(self, task_queue, delay=0):
        super(TaskWorker, self).__init__()
        self.daemon = True

        self._task_queue = task_queue
        self._delay = delay

        self.start()

    def run(self):
        """ Repeatedly get tasks from the TaskQueue and execute them. """
        try:
            while True:
                sleep(self._delay)
                task = self._task_queue.get()
                task._run()
        except:
            self._task_queue._remove_worker()

DEFAULT_TASK_QUEUE = TaskQueue()
<|endoftext|>