<commit_before># Python script to migrate all ECAS data to LabCAS

import os
from xml.etree import ElementTree as ET
import pprint
import glob
import urllib.parse
from shutil import copyfile
import xml.sax.saxutils as saxutils

ecas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/datasets/"
ecas_data_dir = "/data/archive"
labcas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/ecas-metadata/"
labcas_data_dir = "/home/cinquini/ECAS_MIGRATION/labcas_archive"

# RDF streams
sites_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/sites.rdf"
leadpis_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/registered-person.rdf"
organs_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/body-systems.rdf"

sites_map = {}
inv_sites_map = {}
leadpis_map = {}
organs_map = {}

pp = pprint.PrettyPrinter(indent=4)

collection_dict = {
                   "CollectionName":"",
                   "CollectionDescription":"",
                   "Discipline":"",
                   "Institution":"",
                   "InstitutionId":"",
                   "LeadPI":"",
                   "LeadPIId":"",
                   "DataCustodian":"",
                   "DataCustodianEmail":"",
                   "Organ":"",
                   "OrganId":"",
                   "OwnerPrincipal":"cn=All Users,dc=edrn,dc=jpl,dc=nasa,dc=gov",
                   "CollaborativeGroup":"",
                   "QAState":"",
                   "Consortium":"EDRN",
                   "ProtocolName":"",
                   "ProtocolId":"",
                   "Species":"",
                   "MethodDetails":"",
                   "ResultsAndConclusionSummary":"",
                   "PubMedID":"",
                   "DateDatasetFrozen":"",
                   "QAState":"",
                   "DataDisclaimer":""
                   }

dataset_dict = {
                "DatasetId":"",
                "DatasetName":"",
                "DatasetDescription":""
                }

def read_sites_from_rdf(rdf_filepath, metadata_map, inv_metadata_map):
    '''
    Parse an RDF file to populate metadata mappings
    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://edrn.nci.nih.gov/rdf/schema.rdf#',
                  'ns2':'http://purl.org/dc/terms/'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags:
    # <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/sites/313">
    #     <ns2:title>CRUK Cambridge Research Institute</ns2:title>
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        title_element = description_element.find(".//ns2:title", namespaces)
        metadata_map[title_element.text] = rdf_id

    # reverse sites mapping since source metadata is not consistent
    for k in metadata_map:
        inv_metadata_map[metadata_map[k]] = k

def read_organs_from_rdf(rdf_filepath, metadata_map):
    '''
    Parses the RDF file containing organs information
    to map the organ title to the organ id.

    Example XML snippet:
    <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/body-systems/32">
      <ns1:title>Thyroid</ns1:title>
      <rdf:type rdf:resource="http://edrn.nci.nih.gov/rdf/types.rdf#BodySystem"/>
    </rdf:Description>
    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://purl.org/dc/terms/'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        title_element = description_element.find(".//ns1:title", namespaces)
        metadata_map[title_element.text] = rdf_id

def read_leadpis_from_rdf(rdf_filepath, metadata_map):
    '''
    Reads LeadPIs information from the RDF file, maps last name + first name to id

    Example XML snippet:

    <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/registered-person/1645">
      <ns2:surname>Lokshin</ns2:surname>
      <ns2:givenname>Anna</ns2:givenname>
    </rdf:Description>

    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://edrn.nci.nih.gov/rdf/schema.rdf#',
                  'ns2':'http://xmlns.com/foaf/0.1/',
                  'ns3':'http://www.w3.org/2001/vcard-rdf/3.0#'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        lastname_element = description_element.find(".//ns2:surname", namespaces)
        firstname_element = description_element.find(".//ns2:givenname", namespaces)
        metadata_map["%s %s" %(firstname_element.text, lastname_element.text)] = rdf_id

def read_product_type_metadata(input_xml_file):
    '''
    Reads product type metadata from XML and converts it to Python dictionaries.
    '''

    print("Reading %s" % input_xml_file)
    collection_metadata = collection_dict.copy()
    dataset_metadata = dataset_dict.copy()

    xml = ET.parse(input_xml_file)
    root_element = xml.getroot()

    # Description --> CollectionDescription
    description_element = root_element.find('.//description')
    val = cleanup_text(description_element.text)
    collection_metadata['CollectionDescription'] = val

    metadata_element = root_element.find('.//metadata')
    for keyval_element in metadata_element.findall('./keyval'):
        key = keyval_element.find("key").text
        val = keyval_element.find("val").text

        # replace newline characters from metadata values
        if val:
            val = cleanup_text(val)
            # un-escape XML characters
            val = saxutils.unescape(val)

        # DataSetName --> CollectionName, DatasetName
        if key == 'DataSetName':
            collection_metadata['CollectionName'] = val
            collection_metadata['CollectionId'] = val.replace(' ', '_')
            dataset_metadata['DatasetName'] = val
            dataset_metadata['DatasetDescription'] = val
            dataset_metadata['DatasetId'] = val.replace(" ","_")

        # ProtocolID --> ProtocolId, ProtocolName
        elif key == 'ProtocolID' or key == 'ProtocolId':
            collection_metadata['ProtocolId'] = val

        elif key == 'ProtocolName':
            collection_metadata['ProtocolName'] = val

        # LeadPI --> LeadPI, LeadPIId
        elif key == 'LeadPI':
            collection_metadata['LeadPI'] = val

            if leadpis_map.get(val, None):
               collection_metadata['LeadPIId'] = leadpis_map[val]

        # SiteName --> Institution, InstitutionId
        elif key == 'SiteName':
            collection_metadata['Institution'] = val

            # try institution title --> institution id
            if sites_map.get(val, None):
               collection_metadata['InstitutionId'] = sites_map[val]

            # reverse mapping: institution id --> institution title
            elif inv_sites_map.get(val, None):
                collection_metadata['InstitutionId'] = val
                collection_metadata['Institution'] = inv_sites_map[val]

        # DataCustodian --> DataCustodian
        elif key == 'DataCustodian':
            collection_metadata['DataCustodian'] = val

        # DataCustodianEmail --> DataCustodianEmail
        elif key == 'DataCustodianEmail':
            collection_metadata['DataCustodianEmail'] = val

        # OrganSite --> Organ, OrganId
        elif key == 'OrganSite':
            collection_metadata['Organ'] = val

            if organs_map.get(val, None):
               collection_metadata['OrganId'] = organs_map[val]

        # CollaborativeGroup --> CollaborativeGroup
        elif key == 'CollaborativeGroup':
            collection_metadata['CollaborativeGroup'] = val

        # MethodDetails --> MethodDetails
        elif key == 'MethodDetails':
            collection_metadata['MethodDetails'] = val

        # ResultsAndConclusionSummary --> ResultsAndConclusionSummary
        elif key == 'ResultsAndConclusionSummary':
            collection_metadata['ResultsAndConclusionSummary'] = val

        # PubMedID --> PubMedID
        elif key == 'PubMedID':
            collection_metadata['PubMedID'] = "http://www.ncbi.nlm.nih.gov/pubmed/%s" % val

        # DateDatasetFrozen --> DateDatasetFrozen
        elif key == 'DateDatasetFrozen':
            collection_metadata['DateDatasetFrozen'] = val

        # Date --> Date
        elif key == 'Date':
            collection_metadata['Date'] = val

        # QAState --> QAState
        elif key == 'QAState':
            collection_metadata['QAState'] = val

        # DataDisclaimer --> DataDisclaimer
        elif key == 'DataDisclaimer':
            collection_metadata['DataDisclaimer'] = val

    #pp.pprint(collection_metadata)
    #pp.pprint(dataset_metadata)

    return {
             'Collection':collection_metadata,
             'Dataset': dataset_metadata
             }

def write_product_type_metadata(metadata):
    '''
    Serializes metadata from Python dictionaries into a Python configuration file.
    '''

    collection_id = metadata['Collection']['CollectionId']
    config_file_dir = "%s/%s" % (labcas_metadata_dir, collection_id)
    if not os.path.exists(config_file_dir):
        os.makedirs(config_file_dir)
    config_file_name = collection_id + ".cfg"
    config_file_full_name = os.path.join(config_file_dir, config_file_name)

    with open(config_file_full_name, 'w') as f:

        # collection metadata
        f.write('[Collection]\n')
        for key, value in metadata['Collection'].items():
            f.write('%s=%s\n' % ( key, value))

        # dataset metadata
        f.write('[Dataset]\n')
        for key, value in metadata['Dataset'].items():
            f.write('%s=%s\n' % ( key, value))

def read_product_metadata(dataset_dir):

    # list metadata files in all sub-direcories
    met_files = glob.glob('%s/**/*.met' % dataset_dir, recursive=True)

    file_metadata_array = []

    for met_file in met_files:

        file_metadata = {}

        xml = ET.parse(met_file)
        root_element = xml.getroot()

        for keyval_element in root_element.findall('./keyval'):
            key = keyval_element.find("key").text
            val = keyval_element.find("val").text
            if val:
                val = cleanup_text(val)
                # reverse URL encoding
                val = urllib.parse.unquote(val).replace('+',' ')
                # un-escape XML characters
                val = saxutils.unescape(val)

                if key == 'CAS.ProductName' or key == 'CAS.ProductId' or key == 'CAS.ProductReceivedTime':
                    pass # ignore
                elif key == 'FileLocation':
                    pass # ignore since this is the old value and the new value will be added at publishing time
                else:
                    file_metadata[key] = val

        #pp.pprint(file_metadata)

        # add dictionary to list, if not empty
        if file_metadata:
            file_metadata_array.append(file_metadata)

    return file_metadata_array

def copy_products(collection_id, file_metadata_array, output_dir):

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    for file_metadata in file_metadata_array:

        filename = file_metadata['Filename']
        output_file = os.path.join(output_dir, filename)

        input_file = find_most_recent_file(ecas_data_dir, filename)
        if input_file:

            # copy file
            print("Copying file: %s --> %s" % (input_file, output_file))
            copyfile(input_file, output_file)

            # write associated metadata
            write_product_metadata(file_metadata, output_file)

        else:
            print("Cannot find file: '%s' [collection: %s]" % (filename, collection_id) )

def write_product_metadata(file_metadata, output_file):

    met_file = output_file + ".xmlmet"
    with open(met_file, 'w') as f:
        f.write('<cas:metadata xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas">\n')

        # FIXME: select specific elements
        for key, value in file_metadata.items():
            f.write('\t<keyval type="vector">\n')
            f.write('\t\t<key>_File_%s</key>\n' % key)
            f.write('\t\t<val>%s</val>\n' % value)
            f.write('\t</keyval>\n')
        f.write('</cas:metadata>')

def find_most_recent_file(root_dir, file_name):
    result = None
    mtime = None
    for root, dirs, files in os.walk(root_dir):
        if file_name in files:
            file_full_path = os.path.join(root, file_name)
            file_mod_time = os.path.getmtime(file_full_path)
            if mtime:
                if file_mod_time > mtime:
                    result = file_full_path
                    mtime = file_mod_time
            else:
                result = file_full_path
                mtime = file_mod_time
    return result

def cleanup_text(val):
    '''
    Cleans up a text string.
    '''

    # remove new line characters
    val = val.replace('\n',' ').replace('\r\n',' ')
    # remove extra spaces
    val = val.replace("\s+"," ")
    # remove '\' character
    val = val.replace("\ ","")

    return val

if __name__== "__main__":

    # initialize metadata maps with information read from the RDF files
    read_sites_from_rdf(sites_rdf_filepath, sites_map, inv_sites_map)
    read_organs_from_rdf(organs_rdf_filepath, organs_map)
    read_leadpis_from_rdf(leadpis_rdf_filepath, leadpis_map)

    # loop over directories
    filenames = os.listdir(ecas_metadata_dir)

    for filename in filenames:
        dataset_dir = os.path.join(ecas_metadata_dir, filename)
        if os.path.isdir(dataset_dir):

            # FIXME
            #if filename == 'COPY_NUMBER_LEV1':
            #if filename == 'Analysis_of_pancreatic_cancer_biomarkers_in_PLCO_set':
            if filename == 'BCCA_Affy6.0RawData':
            #if True:
                input_xml_file = os.path.join(ecas_metadata_dir, ("%s.met" % filename))

                # read product type metadata from XML, convert to dictionaries
                metadata = read_product_type_metadata(input_xml_file)
                collection_id = metadata['Collection']['CollectionId']
                dataset_id = metadata['Dataset']['DatasetId']

                # write dictionary metadata to collection+dataset configuration file
                write_product_type_metadata(metadata)

                # read product type metadata from XML, convert to dictionaries
                file_metadata_array = read_product_metadata(dataset_dir)

                # copy data files
                # FIXME: remove 1
                output_dir = os.path.join(labcas_data_dir, collection_id, dataset_id, '1')
                copy_products(collection_id, file_metadata_array, output_dir)
<commit_msg>Using RE<commit_after># Python script to migrate all ECAS data to LabCAS

import os
from xml.etree import ElementTree as ET
import pprint
import glob
import urllib.parse
from shutil import copyfile
import xml.sax.saxutils as saxutils
import re

ecas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/datasets/"
ecas_data_dir = "/data/archive"
labcas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/ecas-metadata/"
labcas_data_dir = "/home/cinquini/ECAS_MIGRATION/labcas_archive"

# RDF streams
sites_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/sites.rdf"
leadpis_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/registered-person.rdf"
organs_rdf_filepath = "/home/cinquini/ECAS_MIGRATION/rdf/body-systems.rdf"

sites_map = {}
inv_sites_map = {}
leadpis_map = {}
organs_map = {}

pp = pprint.PrettyPrinter(indent=4)

collection_dict = {
                   "CollectionName":"",
                   "CollectionDescription":"",
                   "Discipline":"",
                   "Institution":"",
                   "InstitutionId":"",
                   "LeadPI":"",
                   "LeadPIId":"",
                   "DataCustodian":"",
                   "DataCustodianEmail":"",
                   "Organ":"",
                   "OrganId":"",
                   "OwnerPrincipal":"cn=All Users,dc=edrn,dc=jpl,dc=nasa,dc=gov",
                   "CollaborativeGroup":"",
                   "QAState":"",
                   "Consortium":"EDRN",
                   "ProtocolName":"",
                   "ProtocolId":"",
                   "Species":"",
                   "MethodDetails":"",
                   "ResultsAndConclusionSummary":"",
                   "PubMedID":"",
                   "DateDatasetFrozen":"",
                   "QAState":"",
                   "DataDisclaimer":""
                   }

dataset_dict = {
                "DatasetId":"",
                "DatasetName":"",
                "DatasetDescription":""
                }

def read_sites_from_rdf(rdf_filepath, metadata_map, inv_metadata_map):
    '''
    Parse an RDF file to populate metadata mappings
    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://edrn.nci.nih.gov/rdf/schema.rdf#',
                  'ns2':'http://purl.org/dc/terms/'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags:
    # <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/sites/313">
    #     <ns2:title>CRUK Cambridge Research Institute</ns2:title>
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        title_element = description_element.find(".//ns2:title", namespaces)
        metadata_map[title_element.text] = rdf_id

    # reverse sites mapping since source metadata is not consistent
    for k in metadata_map:
        inv_metadata_map[metadata_map[k]] = k

def read_organs_from_rdf(rdf_filepath, metadata_map):
    '''
    Parses the RDF file containing organs information
    to map the organ title to the organ id.

    Example XML snippet:
    <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/body-systems/32">
      <ns1:title>Thyroid</ns1:title>
      <rdf:type rdf:resource="http://edrn.nci.nih.gov/rdf/types.rdf#BodySystem"/>
    </rdf:Description>
    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://purl.org/dc/terms/'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        title_element = description_element.find(".//ns1:title", namespaces)
        metadata_map[title_element.text] = rdf_id

def read_leadpis_from_rdf(rdf_filepath, metadata_map):
    '''
    Reads LeadPIs information from the RDF file, maps last name + first name to id

    Example XML snippet:

    <rdf:Description rdf:about="http://edrn.nci.nih.gov/data/registered-person/1645">
      <ns2:surname>Lokshin</ns2:surname>
      <ns2:givenname>Anna</ns2:givenname>
    </rdf:Description>

    '''

    namespaces = {'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                  'ns1':'http://edrn.nci.nih.gov/rdf/schema.rdf#',
                  'ns2':'http://xmlns.com/foaf/0.1/',
                  'ns3':'http://www.w3.org/2001/vcard-rdf/3.0#'}

    print("Reading %s" % rdf_filepath)

    xml = ET.parse(rdf_filepath)
    root_element = xml.getroot()

    # loop over tags
    for description_element in root_element.findall('.//rdf:Description', namespaces):
        about_att = description_element.attrib.get('{%s}about' % namespaces['rdf'])
        rdf_id = about_att.split('/')[-1]
        lastname_element = description_element.find(".//ns2:surname", namespaces)
        firstname_element = description_element.find(".//ns2:givenname", namespaces)
        metadata_map["%s %s" %(firstname_element.text, lastname_element.text)] = rdf_id

def read_product_type_metadata(input_xml_file):
    '''
    Reads product type metadata from XML and converts it to Python dictionaries.
    '''

    print("Reading %s" % input_xml_file)
    collection_metadata = collection_dict.copy()
    dataset_metadata = dataset_dict.copy()

    xml = ET.parse(input_xml_file)
    root_element = xml.getroot()

    # Description --> CollectionDescription
    description_element = root_element.find('.//description')
    val = cleanup_text(description_element.text)
    collection_metadata['CollectionDescription'] = val

    metadata_element = root_element.find('.//metadata')
    for keyval_element in metadata_element.findall('./keyval'):
        key = keyval_element.find("key").text
        val = keyval_element.find("val").text

        # replace newline characters from metadata values
        if val:
            val = cleanup_text(val)
            # un-escape XML characters
            val = saxutils.unescape(val)

        # DataSetName --> CollectionName, DatasetName
        if key == 'DataSetName':
            collection_metadata['CollectionName'] = val
            collection_metadata['CollectionId'] = val.replace(' ', '_')
            dataset_metadata['DatasetName'] = val
            dataset_metadata['DatasetDescription'] = val
            dataset_metadata['DatasetId'] = val.replace(" ","_")

        # ProtocolID --> ProtocolId, ProtocolName
        elif key == 'ProtocolID' or key == 'ProtocolId':
            collection_metadata['ProtocolId'] = val

        elif key == 'ProtocolName':
            collection_metadata['ProtocolName'] = val

        # LeadPI --> LeadPI, LeadPIId
        elif key == 'LeadPI':
            collection_metadata['LeadPI'] = val

            if leadpis_map.get(val, None):
               collection_metadata['LeadPIId'] = leadpis_map[val]

        # SiteName --> Institution, InstitutionId
        elif key == 'SiteName':
            collection_metadata['Institution'] = val

            # try institution title --> institution id
            if sites_map.get(val, None):
               collection_metadata['InstitutionId'] = sites_map[val]

            # reverse mapping: institution id --> institution title
            elif inv_sites_map.get(val, None):
                collection_metadata['InstitutionId'] = val
                collection_metadata['Institution'] = inv_sites_map[val]

        # DataCustodian --> DataCustodian
        elif key == 'DataCustodian':
            collection_metadata['DataCustodian'] = val

        # DataCustodianEmail --> DataCustodianEmail
        elif key == 'DataCustodianEmail':
            collection_metadata['DataCustodianEmail'] = val

        # OrganSite --> Organ, OrganId
        elif key == 'OrganSite':
            collection_metadata['Organ'] = val

            if organs_map.get(val, None):
               collection_metadata['OrganId'] = organs_map[val]

        # CollaborativeGroup --> CollaborativeGroup
        elif key == 'CollaborativeGroup':
            collection_metadata['CollaborativeGroup'] = val

        # MethodDetails --> MethodDetails
        elif key == 'MethodDetails':
            collection_metadata['MethodDetails'] = val

        # ResultsAndConclusionSummary --> ResultsAndConclusionSummary
        elif key == 'ResultsAndConclusionSummary':
            collection_metadata['ResultsAndConclusionSummary'] = val

        # PubMedID --> PubMedID
        elif key == 'PubMedID':
            collection_metadata['PubMedID'] = "http://www.ncbi.nlm.nih.gov/pubmed/%s" % val

        # DateDatasetFrozen --> DateDatasetFrozen
        elif key == 'DateDatasetFrozen':
            collection_metadata['DateDatasetFrozen'] = val

        # Date --> Date
        elif key == 'Date':
            collection_metadata['Date'] = val

        # QAState --> QAState
        elif key == 'QAState':
            collection_metadata['QAState'] = val

        # DataDisclaimer --> DataDisclaimer
        elif key == 'DataDisclaimer':
            collection_metadata['DataDisclaimer'] = val

    #pp.pprint(collection_metadata)
    #pp.pprint(dataset_metadata)

    return {
             'Collection':collection_metadata,
             'Dataset': dataset_metadata
             }

def write_product_type_metadata(metadata):
    '''
    Serializes metadata from Python dictionaries into a Python configuration file.
    '''

    collection_id = metadata['Collection']['CollectionId']
    config_file_dir = "%s/%s" % (labcas_metadata_dir, collection_id)
    if not os.path.exists(config_file_dir):
        os.makedirs(config_file_dir)
    config_file_name = collection_id + ".cfg"
    config_file_full_name = os.path.join(config_file_dir, config_file_name)

    with open(config_file_full_name, 'w') as f:

        # collection metadata
        f.write('[Collection]\n')
        for key, value in metadata['Collection'].items():
            f.write('%s=%s\n' % ( key, value))

        # dataset metadata
        f.write('[Dataset]\n')
        for key, value in metadata['Dataset'].items():
            f.write('%s=%s\n' % ( key, value))

def read_product_metadata(dataset_dir):

    # list metadata files in all sub-direcories
    met_files = glob.glob('%s/**/*.met' % dataset_dir, recursive=True)

    file_metadata_array = []

    for met_file in met_files:

        file_metadata = {}

        xml = ET.parse(met_file)
        root_element = xml.getroot()

        for keyval_element in root_element.findall('./keyval'):
            key = keyval_element.find("key").text
            val = keyval_element.find("val").text
            if val:
                val = cleanup_text(val)
                # reverse URL encoding
                val = urllib.parse.unquote(val).replace('+',' ')
                # un-escape XML characters
                val = saxutils.unescape(val)

                if key == 'CAS.ProductName' or key == 'CAS.ProductId' or key == 'CAS.ProductReceivedTime':
                    pass # ignore
                elif key == 'FileLocation':
                    pass # ignore since this is the old value and the new value will be added at publishing time
                else:
                    file_metadata[key] = val

        #pp.pprint(file_metadata)

        # add dictionary to list, if not empty
        if file_metadata:
            file_metadata_array.append(file_metadata)

    return file_metadata_array

def copy_products(collection_id, file_metadata_array, output_dir):

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    for file_metadata in file_metadata_array:

        filename = file_metadata['Filename']
        output_file = os.path.join(output_dir, filename)

        input_file = find_most_recent_file(ecas_data_dir, filename)
        if input_file:

            # copy file
            print("Copying file: %s --> %s" % (input_file, output_file))
            copyfile(input_file, output_file)

            # write associated metadata
            write_product_metadata(file_metadata, output_file)

        else:
            print("Cannot find file: '%s' [collection: %s]" % (filename, collection_id) )

def write_product_metadata(file_metadata, output_file):

    met_file = output_file + ".xmlmet"
    with open(met_file, 'w') as f:
        f.write('<cas:metadata xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas">\n')

        # FIXME: select specific elements
        for key, value in file_metadata.items():
            f.write('\t<keyval type="vector">\n')
            f.write('\t\t<key>_File_%s</key>\n' % key)
            f.write('\t\t<val>%s</val>\n' % value)
            f.write('\t</keyval>\n')
        f.write('</cas:metadata>')

def find_most_recent_file(root_dir, file_name):
    result = None
    mtime = None
    for root, dirs, files in os.walk(root_dir):
        if file_name in files:
            file_full_path = os.path.join(root, file_name)
            file_mod_time = os.path.getmtime(file_full_path)
            if mtime:
                if file_mod_time > mtime:
                    result = file_full_path
                    mtime = file_mod_time
            else:
                result = file_full_path
                mtime = file_mod_time
    return result

def cleanup_text(val):
    '''
    Cleans up a text string.
    '''

    # remove new line characters
    val = val.replace('\n',' ').replace('\r\n',' ')
    # remove extra spaces
    val = re.sub("\s+"," ",val)
    # remove '\' character
    val = val.replace("\ ","")

    return val

if __name__== "__main__":

    # initialize metadata maps with information read from the RDF files
    read_sites_from_rdf(sites_rdf_filepath, sites_map, inv_sites_map)
    read_organs_from_rdf(organs_rdf_filepath, organs_map)
    read_leadpis_from_rdf(leadpis_rdf_filepath, leadpis_map)

    # loop over directories
    filenames = os.listdir(ecas_metadata_dir)

    for filename in filenames:
        dataset_dir = os.path.join(ecas_metadata_dir, filename)
        if os.path.isdir(dataset_dir):

            # FIXME
            #if filename == 'COPY_NUMBER_LEV1':
            #if filename == 'Analysis_of_pancreatic_cancer_biomarkers_in_PLCO_set':
            if filename == 'BCCA_Affy6.0RawData':
            #if True:
                input_xml_file = os.path.join(ecas_metadata_dir, ("%s.met" % filename))

                # read product type metadata from XML, convert to dictionaries
                metadata = read_product_type_metadata(input_xml_file)
                collection_id = metadata['Collection']['CollectionId']
                dataset_id = metadata['Dataset']['DatasetId']

                # write dictionary metadata to collection+dataset configuration file
                write_product_type_metadata(metadata)

                # read product type metadata from XML, convert to dictionaries
                file_metadata_array = read_product_metadata(dataset_dir)

                # copy data files
                # FIXME: remove 1
                output_dir = os.path.join(labcas_data_dir, collection_id, dataset_id, '1')
                copy_products(collection_id, file_metadata_array, output_dir)
<|endoftext|>


<commit_before>import sqlite3
import csv
import os
import datetime
import re
import string
from HTMLParser import HTMLParser
from badgesDict import badges

def getUsersAnswersAcceptedQuery(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersAnswersAccepted FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Posts.OwnerUserId = " + user + " AND date(Votes.CreationDate) < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

def getUsersQuestionsAcceptedQuery(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersQuestionsAccepted FROM Posts INNER JOIN Votes ON Posts.AcceptedAnswerId = Votes.PostId WHERE Posts.PostTypeId = 1 AND Posts.AcceptedAnswerId IS NOT NULL  AND Posts.OwnerUserId = " + user + " AND date(Votes.CreationDate) < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

def getUsersQuestionsAcceptedQuery2(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersQuestionsAccepted FROM Posts INNER JOIN Votes ON Posts.AcceptedAnswerId = Votes.PostId WHERE Posts.PostTypeId = 1 AND Posts.AcceptedAnswerId IS NOT NULL  AND Posts.OwnerUserId = " + user + " AND Votes.CreationDate < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

# Estrae il numero di upvotes, prima di una certa data (@Date), ottenuti dalle domande postate da un certo utente (@User) */
def getQuestUpVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UpVotesQuest FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 1 AND Votes.VoteTypeId = 2 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di downvotes, prima di una certa data (@Date), ottenuti dalle domande postate da un certo utente (@User) */
def getQuestDownVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS DownVotesQuest FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 1 AND Votes.VoteTypeId = 3 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di upvotes, prima di una certa data (@Date), ottenuti dalle risposte postate da un certo utente (@User) */
def getAnswUpVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UpVotesAnsw FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Votes.VoteTypeId = 2 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di downvotes, prima di una certa data (@Date), ottenuti dalle risposte postate da un certo utente (@User) */
def getAnswDownVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS DownVotesAnsw FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Votes.VoteTypeId = 3 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae l'insieme dei badge sbloccati da un utente (@User) prima di una certa data (@Date) */
def getBadges(user, date):
	return "SELECT Badges.UserId, Badges.Name FROM Badges WHERE Badges.Date < \'" + date + "\' AND Badges.UserId = " + user

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def del_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

def del_code(html):
	return re.sub('\(.*?)\', '', html)

def clean_body(html):
	return del_tags(del_code(html))

def del_punctuation(text):
	return text.translate(string.maketrans ("" , ""), string.punctuation)

def text_length(text):
	text_cleaned_punc = del_punctuation(text)
	text_cleaned = text_cleaned_punc.split(" ")
	word = 0
	for w in text_cleaned:
		if w == "":
			word = word
		elif w == "\n":
			word = word
		elif w == "\t":
			word = word
		else:
			word += 1
	return word

def execute_param_query(db, query):
	conn = sqlite3.connect(db)
	c = conn.cursor()
	result_set = c.execute(query)
	return result_set

def build_dataset(db, file_name, output_file):
	dict_reader = csv.DictReader(open(file_name, 'r'))

	head = dict_reader.fieldnames
	head.append('Weekday')
	head.append('GMTHour')
	head.append('BodyCleaned')
	head.append('BodyLength')
	head.append('TitleLength')
	head.append('UsersAnswersAccepted')
	head.append('UsersQuestionsAccepted')
	#head.append('UsersQuestionsAccepted2')
	head.append('QuestionScore')
	head.append('AnswerScore')
	head.append('BronzeBadge')
	head.append('SilverBadge')
	head.append('GoldBadge')
	head.append('CodeSnippet')
	head.append('Accepted')
	dict_writer = csv.DictWriter(open(output_file, 'w'), head)
	dict_writer.writerow(dict((fn,fn) for fn in head)) #Scrive gli header
	DayL = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']

	p_badges = []

	for row in dict_reader:
		user = row['UserId']
		date = row['PostCreationDate']
		body = row['Body']
		title = row['Title']

		body_cleaned = clean_body(body)
		row['BodyCleaned'] = body_cleaned

		row['TitleLength'] = text_length(title)
		row['BodyLength'] = text_length(body_cleaned)

		print "User: ",user
		print "Date: ",date,"\n"
		if user != 'None':
			result_set1 = execute_param_query(db, getUsersAnswersAcceptedQuery(user, date))
			result_set2 = execute_param_query(db, getUsersQuestionsAcceptedQuery(user, date))
			#result_set3 = execute_param_query(db, getUsersQuestionsAcceptedQuery2(user, date))
			result_set4 = execute_param_query(db, getQuestUpVotes(user, date))
			result_set5 = execute_param_query(db, getQuestDownVotes(user, date))
			result_set6 = execute_param_query(db, getAnswUpVotes(user, date))
			result_set7 = execute_param_query(db, getAnswDownVotes(user, date))
			result_set8 = execute_param_query(db, getBadges(user, date))

			for tup in result_set1:
				row['UsersAnswersAccepted'] = tup[1]
			for tup in result_set2:
				row['UsersQuestionsAccepted'] = tup[1]
			#for tup in result_set3:
			#	row['UsersQuestionsAccepted2'] = tup[1]
			for tup in result_set4:
				q_up = tup[1]
			for tup in result_set5:
				q_down = tup[1]
			for tup in result_set6:
				a_up = tup[1]
			for tup in result_set7:
				a_down = tup[1]

			q_score = q_up - q_down
			a_score = a_up - a_down

			row['QuestionScore'] = q_score
			row['AnswerScore'] = a_score

			bronze = 0
			silver = 0
			gold = 0

			for tup in result_set8:
				if badges.has_key(tup[1]):
					if badges[tup[1]] == 'Bronze':
						bronze += 1
					if badges[tup[1]] == 'Silver':
						silver += 1
					if badges[tup[1]] == 'Gold':
						gold += 1
				else:
					p_badges.append(tup[1])

			row['BronzeBadge'] = bronze
			row['SilverBadge'] = silver
			row['GoldBadge'] = gold

			d = date.split('T')
			dat = d[0].split('-')
			hou = d[1].split(':')
			hour = hou[0]

			row['GMTHour'] = hour

			print "Anno: ", dat[0]
			print "Mese: ", dat[1]
			print "Giorno: ", dat[2]

			weekday = DayL[datetime.date(int(dat[0]),int(dat[1]),int(dat[2])).weekday()] + ""
			row['Weekday'] = weekday

			code_snippet = "no"
			if '' in body:
				code_snippet = "yes"

			row['CodeSnippet'] = code_snippet

			accepted = 'yes'
			if row['PostAcceptedAnswerId'] == 'None':
				accepted = 'no'
			row['Accepted'] = accepted

		dict_writer.writerow(row)

	print p_badges
	return 'Done'

def p_badges(db, file_name):
	dict_reader = csv.DictReader(open(file_name, 'r'))
	new_badges = open("badges.txt", 'w')

	p_badges = []

	for row in dict_reader:
		user = row['UserId']
		date = row['PostCreationDate']

		print "User: ",user
		print "Date: ",date,"\n"
		if user != 'None':

			result_set8 = execute_param_query(db, getBadges(user, date))

			bronze = 0
			silver = 0
			gold = 0

			for tup in result_set8:
				if badges.has_key(tup[1]):
					if badges[tup[1]] == 'Bronze':
						bronze += 1
					if badges[tup[1]] == 'Silver':
						silver += 1
					if badges[tup[1]] == 'Gold':
						gold += 1
				else:
					if tup[1] in p_badges:
						print ""
					else:
						p_badges.append(tup[1])
						new_badges.write(tup[1]+"\n")

			row['BronzeBadge'] = bronze
			row['SilverBadge'] = silver
			row['GoldBadge'] = gold

		#dict_writer.writerow(row)
	new_badges.close()
	print p_badges
	return 'Done'

build_dataset('academia.dump.db', 'result-set.csv', 'test.csv')
#p_badges('academia.dump.db', 'result-set.csv')
#print text_length('As from title. What kind of visa class do I have to apply for, in order to work as an academic in Japan ?')
<commit_msg>script build dataset<commit_after>import sqlite3
import csv
import os
import datetime
import re
import string
import operator
from HTMLParser import HTMLParser
from badgesDict import badges

def getUsersAnswersAcceptedQuery(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersAnswersAccepted FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Posts.OwnerUserId = " + user + " AND date(Votes.CreationDate) < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

def getUsersQuestionsAcceptedQuery(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersQuestionsAccepted FROM Posts INNER JOIN Votes ON Posts.AcceptedAnswerId = Votes.PostId WHERE Posts.PostTypeId = 1 AND Posts.AcceptedAnswerId IS NOT NULL  AND Posts.OwnerUserId = " + user + " AND date(Votes.CreationDate) < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

def getUsersQuestionsAcceptedQuery2(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UsersQuestionsAccepted FROM Posts INNER JOIN Votes ON Posts.AcceptedAnswerId = Votes.PostId WHERE Posts.PostTypeId = 1 AND Posts.AcceptedAnswerId IS NOT NULL  AND Posts.OwnerUserId = " + user + " AND Votes.CreationDate < date(\'" + date + "\') AND Votes.VoteTypeId = 1"

# Estrae il numero di upvotes, prima di una certa data (@Date), ottenuti dalle domande postate da un certo utente (@User) */
def getQuestUpVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UpVotesQuest FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 1 AND Votes.VoteTypeId = 2 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di downvotes, prima di una certa data (@Date), ottenuti dalle domande postate da un certo utente (@User) */
def getQuestDownVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS DownVotesQuest FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 1 AND Votes.VoteTypeId = 3 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di upvotes, prima di una certa data (@Date), ottenuti dalle risposte postate da un certo utente (@User) */
def getAnswUpVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS UpVotesAnsw FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Votes.VoteTypeId = 2 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae il numero di downvotes, prima di una certa data (@Date), ottenuti dalle risposte postate da un certo utente (@User) */
def getAnswDownVotes(user, date):
	return "SELECT Posts.OwnerUserId AS UserId, count(Posts.Id) AS DownVotesAnsw FROM Posts INNER JOIN Votes ON Posts.Id = Votes.PostId WHERE Posts.PostTypeId = 2 AND Votes.VoteTypeId = 3 AND date(Votes.CreationDate) < date(\'" + date + "\') AND Posts.OwnerUserId = " + user

# Estrae l'insieme dei badge sbloccati da un utente (@User) prima di una certa data (@Date) */
def getBadges(user, date):
	return "SELECT Badges.UserId, Badges.Name FROM Badges WHERE Badges.Date < \'" + date + "\' AND Badges.UserId = " + user

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def del_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

def del_code(html):
	return re.sub('\(.*?)\', '', html)

def clean_body(html):
	return del_tags(del_code(html))

def del_punctuation(text):
	return text.translate(string.maketrans ("" , ""), string.punctuation)

def text_length(text):
	text_cleaned_punc = del_punctuation(text)
	text_cleaned = text_cleaned_punc.split(" ")
	word = 0
	for w in text_cleaned:
		if w == "":
			word = word
		elif w == "\n":
			word = word
		elif w == "\t":
			word = word
		else:
			word += 1
	return word

# Restituisce tutti i titoli e tutti i corpi, presenti nel file in input, fusi in un'unica variabile
def get_corpus(file_name):
	dict_reader = csv.DictReader(open(file_name, 'r'))

	corpus = ""

	for row in dict_reader:
		body = row['Body']
		title = row['Title']

		body_cleaned = del_punctuation(clean_body(body))
		title_cleaned = del_punctuation(clean_body(title))

		corpus = corpus + title_cleaned + " " + body_cleaned

	return corpus

def create_dictionary(corpus):
	print "Processing..."
	words = re.findall(r"[\w']+", corpus)
	n_word = 0
	dictionary = {}

	for word in words:
		if word == "":
			n_word = n_word
		elif word == "\n":
			n_word = n_word
		elif word == "\t":
			n_word = n_word
		else:
			n_word += 1
			if dictionary.has_key(word):
				dictionary[word] += 1
			else:
				dictionary[word] = 1

	print "Number of words ", n_word

	for key in dictionary.keys():
		dictionary[key] = dictionary[key]/n_word

	#sorted_dict = sorted(dictionary.iteritems(), key=operator.itemgetter(1))
	sorted_dict = sorted(dictionary, key=dictionary.get, reverse=True)

	#print sorted_dict

	w = csv.writer(open("output.csv", "w"))
	#for key, val in sorted_dict.items():
	#	w.writerow([key, val])
	for elem in sorted_dict:
		w.writerow([elem,dictionary[elem]])

	print "Done..."

	return sorted_dict

def execute_param_query(db, query):
	conn = sqlite3.connect(db)
	c = conn.cursor()
	result_set = c.execute(query)
	return result_set

def build_dataset(db, file_name, output_file):
	dict_reader = csv.DictReader(open(file_name, 'r'))

	head = dict_reader.fieldnames
	head.append('Weekday')
	head.append('GMTHour')
	head.append('BodyCleaned')
	head.append('BodyLength')
	head.append('TitleLength')
	head.append('UsersAnswersAccepted')
	head.append('UsersQuestionsAccepted')
	#head.append('UsersQuestionsAccepted2')
	head.append('QuestionScore')
	head.append('AnswerScore')
	head.append('BronzeBadge')
	head.append('SilverBadge')
	head.append('GoldBadge')
	head.append('CodeSnippet')
	head.append('Accepted')
	dict_writer = csv.DictWriter(open(output_file, 'w'), head)
	dict_writer.writerow(dict((fn,fn) for fn in head)) #Scrive gli header
	DayL = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']

	p_badges = []

	for row in dict_reader:
		user = row['UserId']
		date = row['PostCreationDate']
		body = row['Body']
		title = row['Title']

		body_cleaned = clean_body(body)
		row['BodyCleaned'] = body_cleaned

		row['TitleLength'] = text_length(title)
		row['BodyLength'] = text_length(body_cleaned)

		print "User: ",user
		print "Date: ",date,"\n"
		if user != 'None':
			result_set1 = execute_param_query(db, getUsersAnswersAcceptedQuery(user, date))
			result_set2 = execute_param_query(db, getUsersQuestionsAcceptedQuery(user, date))
			#result_set3 = execute_param_query(db, getUsersQuestionsAcceptedQuery2(user, date))
			result_set4 = execute_param_query(db, getQuestUpVotes(user, date))
			result_set5 = execute_param_query(db, getQuestDownVotes(user, date))
			result_set6 = execute_param_query(db, getAnswUpVotes(user, date))
			result_set7 = execute_param_query(db, getAnswDownVotes(user, date))
			result_set8 = execute_param_query(db, getBadges(user, date))

			for tup in result_set1:
				row['UsersAnswersAccepted'] = tup[1]
			for tup in result_set2:
				row['UsersQuestionsAccepted'] = tup[1]
			#for tup in result_set3:
			#	row['UsersQuestionsAccepted2'] = tup[1]
			for tup in result_set4:
				q_up = tup[1]
			for tup in result_set5:
				q_down = tup[1]
			for tup in result_set6:
				a_up = tup[1]
			for tup in result_set7:
				a_down = tup[1]

			q_score = q_up - q_down
			a_score = a_up - a_down

			row['QuestionScore'] = q_score
			row['AnswerScore'] = a_score

			bronze = 0
			silver = 0
			gold = 0

			for tup in result_set8:
				if badges.has_key(tup[1]):
					if badges[tup[1]] == 'Bronze':
						bronze += 1
					if badges[tup[1]] == 'Silver':
						silver += 1
					if badges[tup[1]] == 'Gold':
						gold += 1
				else:
					p_badges.append(tup[1])

			row['BronzeBadge'] = bronze
			row['SilverBadge'] = silver
			row['GoldBadge'] = gold

			d = date.split('T')
			dat = d[0].split('-')
			hou = d[1].split(':')
			hour = hou[0]

			row['GMTHour'] = hour

			print "Anno: ", dat[0]
			print "Mese: ", dat[1]
			print "Giorno: ", dat[2]

			weekday = DayL[datetime.date(int(dat[0]),int(dat[1]),int(dat[2])).weekday()] + ""
			row['Weekday'] = weekday

			code_snippet = "no"
			if '' in body:
				code_snippet = "yes"

			row['CodeSnippet'] = code_snippet

			accepted = 'yes'
			if row['PostAcceptedAnswerId'] == 'None':
				accepted = 'no'
			row['Accepted'] = accepted

		dict_writer.writerow(row)

	print p_badges
	return 'Done'

def p_badges(db, file_name):
	dict_reader = csv.DictReader(open(file_name, 'r'))
	new_badges = open("badges.txt", 'w')

	p_badges = []

	for row in dict_reader:
		user = row['UserId']
		date = row['PostCreationDate']

		print "User: ",user
		print "Date: ",date,"\n"
		if user != 'None':

			result_set8 = execute_param_query(db, getBadges(user, date))

			bronze = 0
			silver = 0
			gold = 0

			for tup in result_set8:
				if badges.has_key(tup[1]):
					if badges[tup[1]] == 'Bronze':
						bronze += 1
					if badges[tup[1]] == 'Silver':
						silver += 1
					if badges[tup[1]] == 'Gold':
						gold += 1
				else:
					if tup[1] in p_badges:
						print ""
					else:
						p_badges.append(tup[1])
						new_badges.write(tup[1]+"\n")

			row['BronzeBadge'] = bronze
			row['SilverBadge'] = silver
			row['GoldBadge'] = gold

		#dict_writer.writerow(row)
	new_badges.close()
	print p_badges
	return 'Done'

create_dictionary(get_corpus('result-set.csv'))
#build_dataset('academia.dump.db', 'result-set.csv', 'test.csv')
#p_badges('academia.dump.db', 'result-set.csv')
#print text_length('As from title. What kind of visa class do I have to apply for, in order to work as an academic in Japan ?')
<|endoftext|>


<commit_before>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.core.mail import send_mail
from django.db.models import Count, Max
from django.template.loader import get_template, render_to_string
from django.utils.translation import activate, ungettext
from django.utils.translation import ugettext as _

from celery.task import periodic_task, task

@task(ignore_result=True)
def remove_empty_groups():
    """Remove empty groups."""

    from mozillians.groups.models import Group, Skill

    for model in [Group, Skill]:
        model.objects.annotate(mcount=Count('members')).filter(mcount=0).delete()

# TODO: Schedule this task nightly

@task(ignore_result=True)
def send_pending_membership_emails():
    """
    For each curated group that has pending memberships that the curators have
    not yet been emailed about, send to all the curators an email with the count
    of all pending memberships and a link to view and manage the requests.
    """

    from mozillians.groups.models import Group, GroupMembership

    # Curated groups that have pending membership requests
    groups = Group.objects.exclude(curators__isnull=True)
    groups = groups.filter(groupmembership__status=GroupMembership.PENDING).distinct()

    for group in groups:
        # what's the max pk of pending memberships?
        pending_memberships = group.groupmembership_set.filter(status=GroupMembership.PENDING)
        max_pk = pending_memberships.aggregate(max_pk=Max('pk'))['max_pk']
        # Only send reminder if there are newer requests than we'd previously reminded about
        if max_pk > group.max_reminder:
            # TODO: Switch locale to curator's preferred language so translation will occur
            # Using English for now
            activate('en-us')

            count = pending_memberships.count()
            subject = ungettext(
                '%(count)d outstanding request to join Mozillians group "%(name)s"',
                '%(count)d outstanding requests to join Mozillians group "%(name)s"',
                count
            ) % {
                'count': count,
                'name': group.name
            }
            body = render_to_string('groups/email/memberships_pending.txt', {
                'group': group,
                'count': count,
            })

            send_mail(subject, body, settings.FROM_NOREPLY,
                      [profile.user.email for profile in group.curators.all()],
                      fail_silently=False)

            group.max_reminder = max_pk
            group.save()

@task(ignore_result=True)
def email_membership_change(group_pk, user_pk, old_status, new_status):
    """
    Email user that their group membership status has changed.

    old_status and new_status can either be a valid value for GroupMembership.status,
    or None if we're going from or to a state where there is no GroupMembership
    record (e.g. if they're being removed from a group).

    This is queued from Group.add_member() and Group.remove_member().
    """

    from mozillians.groups.models import Group, GroupMembership

    group = Group.objects.get(pk=group_pk)
    user = User.objects.get(pk=user_pk)

    # TODO: Switch locale to user's preferred language so translation will occur
    # Using English for now
    activate('en-us')

    if old_status in [GroupMembership.PENDING, GroupMembership.PENDING_TERMS]:
        if new_status == GroupMembership.MEMBER:
            subject = _('Accepted to Mozillians group "%s"') % group.name
            template_name = 'groups/email/accepted.txt'
        elif new_status is None:
            subject = _('Not accepted to Mozillians group "%s"') % group.name
            template_name = 'groups/email/rejected.txt'
        else:
            # Odd things happen in some of our tests. Don't worry about it.
            raise ValueError('BAD ARGS TO email_membership_change')
    else:
        if new_status in [GroupMembership.PENDING_TERMS, GroupMembership.PENDING]:
            subject = _('Status changed for Mozillians group "%s"') % group.name
            template_name = 'groups/email/membership_status_changed.txt'
        else:
            # The user is a member that was removed from the group.
            subject = _('Removed from Mozillians group "%s"') % group.name
            template_name = 'groups/email/member_removed.txt'

    context = {
        'group': group,
        'user': user,
    }
    template = get_template(template_name)
    body = template.render(context)
    send_mail(subject, body, settings.FROM_NOREPLY, [user.email], fail_silently=False)

@periodic_task(run_every=timedelta(hours=24))
def invalidate_group_membership():
    """
    For groups with defined `invalidation_days` we need to invalidate
    user membership after timedelta.
    """
    from mozillians.groups.models import Group, GroupMembership

    groups = Group.objects.filter(invalidation_days__isnull=False)

    for group in groups:
        curator_ids = group.curators.all().values_list('id', flat=True)
        last_update = datetime.now() - timedelta(days=group.invalidation_days)
        memberships = (group.groupmembership_set.filter(status=GroupMembership.MEMBER)
                                                .filter(updated_on__lte=last_update)
                                                .exclude(userprofile__id__in=curator_ids))

        for member in memberships:
            status = None
            if group.accepting_new_members != Group.OPEN:
                status = GroupMembership.PENDING
            group.remove_member(member.userprofile, status=status)

@task(ignore_result=True)
def notify_redeemer_invitation(pk, custom_text=''):

    from mozillians.groups.models import Invite

    invite = Invite.objects.get(pk=pk)
    subject_msg = unicode('[Mozillians] You have been invited to join group "{0}"')
    subject = _(subject_msg.format(invite.group.name))
    template = get_template('groups/email/invite_email.txt')
    ctx = {
        'inviter_full_name': invite.inviter.full_name,
        'redeemer_full_name': invite.redeemer.full_name,
        'group_name': invite.group.name,
        'group_url': invite.group.get_absolute_url(),
        'custom_text': custom_text
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [invite.redeemer.email])

@task(ignore_result=True)
def notify_curators_invitation_accepted(pk):

    from mozillians.groups.models import Invite

    invite = Invite.objects.get(pk=pk)
    subject_msg = unicode('[Mozillians] {0} has accepted your invitation to join group "{1}"')
    subject = _(subject_msg.format(invite.redeemer.full_name, invite.group.name))
    template = get_template('groups/email/invite_accepted_email.txt')
    ctx = {
        'inviter_full_name': invite.inviter.full_name,
        'redeemer_full_name': invite.redeemer.full_name,
        'group_name': invite.group.name,
        'group_url': invite.group.get_absolute_url()
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [invite.inviter.email])

@task(ignore_result=True)
def notify_curators_invitation_rejected(redeemer_pk, inviter_pk, group_pk):

    from mozillians.groups.models import Group
    from mozillians.users.models import UserProfile

    inviter = UserProfile.objects.get(pk=inviter_pk)
    redeemer = UserProfile.objects.get(pk=redeemer_pk)
    group = Group.objects.get(pk=group_pk)

    subject_msg = unicode('[Mozillians] {0} has rejected your invitation to join group "{1}"')
    subject = _(subject_msg.format(redeemer.full_name, group.name))
    template = get_template('groups/email/invite_rejected_email.txt')
    ctx = {
        'redeemer_full_name': redeemer.full_name,
        'inviter_full_name': inviter.full_name,
        'group_name': group.name
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [inviter.email])

@task(ignore_result=True)
def notify_redeemer_invitation_invalid(redeemer_pk, group_pk):

    from mozillians.groups.models import Group
    from mozillians.users.models import UserProfile

    group = Group.objects.get(pk=group_pk)
    redeemer = UserProfile.objects.get(pk=redeemer_pk)
    subject_msg = '[Mozillians] Invitation to group "{0}" is no longer valid'
    subject = _(subject_msg.format(group.name))
    template = get_template('groups/email/invite_invalid_email.txt')
    ctx = {
        'group_name': group.name,
        'redeemer_full_name': redeemer.full_name
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [redeemer.email])
<commit_msg>[bug 1331590] Invalidate even members with PENDING/PENDING_TERMS status.<commit_after>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.core.mail import send_mail
from django.db.models import Count, Max
from django.template.loader import get_template, render_to_string
from django.utils.translation import activate, ungettext
from django.utils.translation import ugettext as _

from celery.task import periodic_task, task

@task(ignore_result=True)
def remove_empty_groups():
    """Remove empty groups."""

    from mozillians.groups.models import Group, Skill

    for model in [Group, Skill]:
        model.objects.annotate(mcount=Count('members')).filter(mcount=0).delete()

# TODO: Schedule this task nightly

@task(ignore_result=True)
def send_pending_membership_emails():
    """
    For each curated group that has pending memberships that the curators have
    not yet been emailed about, send to all the curators an email with the count
    of all pending memberships and a link to view and manage the requests.
    """

    from mozillians.groups.models import Group, GroupMembership

    # Curated groups that have pending membership requests
    groups = Group.objects.exclude(curators__isnull=True)
    groups = groups.filter(groupmembership__status=GroupMembership.PENDING).distinct()

    for group in groups:
        # what's the max pk of pending memberships?
        pending_memberships = group.groupmembership_set.filter(status=GroupMembership.PENDING)
        max_pk = pending_memberships.aggregate(max_pk=Max('pk'))['max_pk']
        # Only send reminder if there are newer requests than we'd previously reminded about
        if max_pk > group.max_reminder:
            # TODO: Switch locale to curator's preferred language so translation will occur
            # Using English for now
            activate('en-us')

            count = pending_memberships.count()
            subject = ungettext(
                '%(count)d outstanding request to join Mozillians group "%(name)s"',
                '%(count)d outstanding requests to join Mozillians group "%(name)s"',
                count
            ) % {
                'count': count,
                'name': group.name
            }
            body = render_to_string('groups/email/memberships_pending.txt', {
                'group': group,
                'count': count,
            })

            send_mail(subject, body, settings.FROM_NOREPLY,
                      [profile.user.email for profile in group.curators.all()],
                      fail_silently=False)

            group.max_reminder = max_pk
            group.save()

@task(ignore_result=True)
def email_membership_change(group_pk, user_pk, old_status, new_status):
    """
    Email user that their group membership status has changed.

    old_status and new_status can either be a valid value for GroupMembership.status,
    or None if we're going from or to a state where there is no GroupMembership
    record (e.g. if they're being removed from a group).

    This is queued from Group.add_member() and Group.remove_member().
    """

    from mozillians.groups.models import Group, GroupMembership

    group = Group.objects.get(pk=group_pk)
    user = User.objects.get(pk=user_pk)

    # TODO: Switch locale to user's preferred language so translation will occur
    # Using English for now
    activate('en-us')

    if old_status in [GroupMembership.PENDING, GroupMembership.PENDING_TERMS]:
        if new_status == GroupMembership.MEMBER:
            subject = _('Accepted to Mozillians group "%s"') % group.name
            template_name = 'groups/email/accepted.txt'
        elif new_status is None:
            subject = _('Not accepted to Mozillians group "%s"') % group.name
            template_name = 'groups/email/rejected.txt'
        else:
            # Odd things happen in some of our tests. Don't worry about it.
            raise ValueError('BAD ARGS TO email_membership_change')
    else:
        if new_status in [GroupMembership.PENDING_TERMS, GroupMembership.PENDING]:
            subject = _('Status changed for Mozillians group "%s"') % group.name
            template_name = 'groups/email/membership_status_changed.txt'
        else:
            # The user is a member that was removed from the group.
            subject = _('Removed from Mozillians group "%s"') % group.name
            template_name = 'groups/email/member_removed.txt'

    context = {
        'group': group,
        'user': user,
    }
    template = get_template(template_name)
    body = template.render(context)
    send_mail(subject, body, settings.FROM_NOREPLY, [user.email], fail_silently=False)

@periodic_task(run_every=timedelta(hours=24))
def invalidate_group_membership():
    """
    For groups with defined `invalidation_days` we need to invalidate
    user membership after timedelta.
    """
    from mozillians.groups.models import Group, GroupMembership

    groups = Group.objects.filter(invalidation_days__isnull=False)

    for group in groups:
        curator_ids = group.curators.all().values_list('id', flat=True)
        last_update = datetime.now() - timedelta(days=group.invalidation_days)
        memberships = (group.groupmembership_set.filter(updated_on__lte=last_update)
                                                .exclude(userprofile__id__in=curator_ids))

        for member in memberships:
            status = None
            if group.accepting_new_members != Group.OPEN:
                status = GroupMembership.PENDING
            group.remove_member(member.userprofile, status=status)

@task(ignore_result=True)
def notify_redeemer_invitation(pk, custom_text=''):

    from mozillians.groups.models import Invite

    invite = Invite.objects.get(pk=pk)
    subject_msg = unicode('[Mozillians] You have been invited to join group "{0}"')
    subject = _(subject_msg.format(invite.group.name))
    template = get_template('groups/email/invite_email.txt')
    ctx = {
        'inviter_full_name': invite.inviter.full_name,
        'redeemer_full_name': invite.redeemer.full_name,
        'group_name': invite.group.name,
        'group_url': invite.group.get_absolute_url(),
        'custom_text': custom_text
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [invite.redeemer.email])

@task(ignore_result=True)
def notify_curators_invitation_accepted(pk):

    from mozillians.groups.models import Invite

    invite = Invite.objects.get(pk=pk)
    subject_msg = unicode('[Mozillians] {0} has accepted your invitation to join group "{1}"')
    subject = _(subject_msg.format(invite.redeemer.full_name, invite.group.name))
    template = get_template('groups/email/invite_accepted_email.txt')
    ctx = {
        'inviter_full_name': invite.inviter.full_name,
        'redeemer_full_name': invite.redeemer.full_name,
        'group_name': invite.group.name,
        'group_url': invite.group.get_absolute_url()
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [invite.inviter.email])

@task(ignore_result=True)
def notify_curators_invitation_rejected(redeemer_pk, inviter_pk, group_pk):

    from mozillians.groups.models import Group
    from mozillians.users.models import UserProfile

    inviter = UserProfile.objects.get(pk=inviter_pk)
    redeemer = UserProfile.objects.get(pk=redeemer_pk)
    group = Group.objects.get(pk=group_pk)

    subject_msg = unicode('[Mozillians] {0} has rejected your invitation to join group "{1}"')
    subject = _(subject_msg.format(redeemer.full_name, group.name))
    template = get_template('groups/email/invite_rejected_email.txt')
    ctx = {
        'redeemer_full_name': redeemer.full_name,
        'inviter_full_name': inviter.full_name,
        'group_name': group.name
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [inviter.email])

@task(ignore_result=True)
def notify_redeemer_invitation_invalid(redeemer_pk, group_pk):

    from mozillians.groups.models import Group
    from mozillians.users.models import UserProfile

    group = Group.objects.get(pk=group_pk)
    redeemer = UserProfile.objects.get(pk=redeemer_pk)
    subject_msg = '[Mozillians] Invitation to group "{0}" is no longer valid'
    subject = _(subject_msg.format(group.name))
    template = get_template('groups/email/invite_invalid_email.txt')
    ctx = {
        'group_name': group.name,
        'redeemer_full_name': redeemer.full_name
    }
    message = template.render(ctx)
    send_mail(subject, message, settings.FROM_NOREPLY, [redeemer.email])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from cmd2 import Cmd, options, make_option
import h5_wrapper
import os

class CmdApp(Cmd):

    def __init__(self):
        Cmd.__init__(self)
        Cmd.shortcuts.update({'!': 'bang', '$': 'shell'})
        self.abbrev = True
        self.prompt = '[] > '

    def precmd(self, line):
        if not line.startswith('load') and \
           (line.endswith('.h5') or line.endswith('.hdf5')):
            line = 'load ' + line
        return line

    def postcmd(self, stop, line):
        if self.explorer:
            self.prompt = '[{}: {}/] > '.format(
                os.path.basename(self.explorer.filename),
                os.path.basename(self.explorer.working_dir))
        return stop

    def do_source(self, args):
        Cmd.do_load(self, args)

    def do_load(self, args, opts=None):
        print("Loading " + args)
        self.explorer = h5_wrapper.H5Explorer(args)

    def do_ls(self, args, opts=None):
        if len(args.strip()) > 0:
            for g in self.explorer.list_groups(args):
                print(g + "/")

            for ds in self.explorer.list_datasets(args):
                print(ds)
        else:
            for g in self.explorer.list_groups():
                print(g + "/")

            for ds in self.explorer.list_datasets():
                print(ds)

    def do_cd(self, args, opts=None):
        if len(args) == 0:
            args = '/'
        self.explorer.change_dir(args)

    def do_pwd(self, args, opts=None):
        print(self.explorer.working_dir)

    def do_pushd(self, args, opts=None):
        self.explorer.push_dir(args)

    def do_popd(self, args, opts=None):
        self.explorer.pop_dir()
        self.do_pwd(None)

    def do_mkdir(self, args, opts=None):
        self.explorer.create_group(args)

    def do_rmdir(self, args, opts=None):
        raise NotImplementedError

    def do_rm(self, args, opts=None):
        raise NotImplementedError

    def do_cat(self, args, opts=None):
        raise NotImplementedError

    def do_head(self, args, opts=None):
        raise NotImplementedError

    def do_tail(self, args, opts=None):
        raise NotImplementedError

    def do_cp(self, args, opts=None):
        raise NotImplementedError

    def do_mv(self, args, opts=None):
        raise NotImplementedError

    def do_bang(self, args, opts=None):
        if args.strip() == '!':
            self.onecmd(self.history[-2])
        elif args.strip().isnumeric():
            print(self.history[-1 * int(args) - 1])
            self.onecmd(self.history[-1 * int(args) - 1])
        else:
            history = self.history.copy()
            history.reverse()
            for cmd in history:
                if cmd.startswith(args):
                    self.onecmd(cmd)
                    return False
            raise ValueError("{}: event not found".format(args))

    def do_clear(self, args):
        os.system('clear')

    def do_exit(self, args):
        return True

if __name__ == '__main__':
    c = CmdApp()
    c.cmdloop()
<commit_msg>(Closes #32) Makes explorer None in init<commit_after>#!/usr/bin/env python3
from cmd2 import Cmd, options, make_option
import h5_wrapper
import os

class CmdApp(Cmd):

    def __init__(self):
        Cmd.__init__(self)
        Cmd.shortcuts.update({'!': 'bang', '$': 'shell'})
        self.abbrev = True
        self.prompt = '[] > '
        self.explorer = None

    def precmd(self, line):
        if not line.startswith('load') and \
           (line.endswith('.h5') or line.endswith('.hdf5')):
            line = 'load ' + line
        return line

    def postcmd(self, stop, line):
        if self.explorer:
            self.prompt = '[{}: {}/] > '.format(
                os.path.basename(self.explorer.filename),
                os.path.basename(self.explorer.working_dir))
        return stop

    def do_source(self, args):
        Cmd.do_load(self, args)

    def do_load(self, args, opts=None):
        print("Loading " + args)
        self.explorer = h5_wrapper.H5Explorer(args)

    def do_ls(self, args, opts=None):
        if len(args.strip()) > 0:
            for g in self.explorer.list_groups(args):
                print(g + "/")

            for ds in self.explorer.list_datasets(args):
                print(ds)
        else:
            for g in self.explorer.list_groups():
                print(g + "/")

            for ds in self.explorer.list_datasets():
                print(ds)

    def do_cd(self, args, opts=None):
        if len(args) == 0:
            args = '/'
        self.explorer.change_dir(args)

    def do_pwd(self, args, opts=None):
        print(self.explorer.working_dir)

    def do_pushd(self, args, opts=None):
        self.explorer.push_dir(args)

    def do_popd(self, args, opts=None):
        self.explorer.pop_dir()
        self.do_pwd(None)

    def do_mkdir(self, args, opts=None):
        self.explorer.create_group(args)

    def do_rmdir(self, args, opts=None):
        raise NotImplementedError

    def do_rm(self, args, opts=None):
        raise NotImplementedError

    def do_cat(self, args, opts=None):
        raise NotImplementedError

    def do_head(self, args, opts=None):
        raise NotImplementedError

    def do_tail(self, args, opts=None):
        raise NotImplementedError

    def do_cp(self, args, opts=None):
        raise NotImplementedError

    def do_mv(self, args, opts=None):
        raise NotImplementedError

    def do_bang(self, args, opts=None):
        if args.strip() == '!':
            self.onecmd(self.history[-2])
        elif args.strip().isnumeric():
            print(self.history[-1 * int(args) - 1])
            self.onecmd(self.history[-1 * int(args) - 1])
        else:
            history = self.history.copy()
            history.reverse()
            for cmd in history:
                if cmd.startswith(args):
                    self.onecmd(cmd)
                    return False
            raise ValueError("{}: event not found".format(args))

    def do_clear(self, args):
        os.system('clear')

    def do_exit(self, args):
        return True

if __name__ == '__main__':
    c = CmdApp()
    c.cmdloop()
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from . import Operation

##  An operation that groups several other operations together.
#
#   The intent of this operation is to hide an underlying chain of operations
#   from the user if they correspond to only one interaction with the user, such
#   as an operation applied to multiple scene nodes or a re-arrangement of
#   multiple items in the scene.
class GroupedOperation(Operation.Operation):
    ##  Creates a new grouped operation.
    #
    #   The grouped operation is empty after its initialisation.
    def __init__(self):
        super().__init__()
        self._children = []
        self._finalised = False #Indicates if this operation is ever used. After that, it may no longer be modified.

    ##  Adds an operation to this group.
    #
    #   The operation will be undone together with the rest of the operations in
    #   this group.
    #   Note that when the order matters, the operations are undone in reverse
    #   order as the order in which they are added.
    def addOperation(self, op):
        if self._finalised:
            raise Exception("A grouped operation may not be modified after it is used.")
        self._children.append(op)

    ##  Undo all operations in this group.
    #
    #   The operations are undone in reverse order as the order in which they
    #   were added.
    def undo(self):
        for op in reversed(self._children):
            op.undo()
        self._finalised = True

    ##  Redoes all operations in this group.
    def redo(self):
        for op in self._children:
            op.redo()
        self._finalised = True<commit_msg>Implement merging of GroupedOperations<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from . import Operation

##  An operation that groups several other operations together.
#
#   The intent of this operation is to hide an underlying chain of operations
#   from the user if they correspond to only one interaction with the user, such
#   as an operation applied to multiple scene nodes or a re-arrangement of
#   multiple items in the scene.
class GroupedOperation(Operation.Operation):
    ##  Creates a new grouped operation.
    #
    #   The grouped operation is empty after its initialisation.
    def __init__(self):
        super().__init__()
        self._children = []
        self._finalised = False #Indicates if this operation is ever used. After that, it may no longer be modified.

    ##  Adds an operation to this group.
    #
    #   The operation will be undone together with the rest of the operations in
    #   this group.
    #   Note that when the order matters, the operations are undone in reverse
    #   order as the order in which they are added.
    def addOperation(self, op):
        if self._finalised:
            raise Exception("A grouped operation may not be modified after it is used.")
        self._children.append(op)

    ##  Undo all operations in this group.
    #
    #   The operations are undone in reverse order as the order in which they
    #   were added.
    def undo(self):
        for op in reversed(self._children):
            op.undo()
        self._finalised = True

    ##  Redoes all operations in this group.
    def redo(self):
        for op in self._children:
            op.redo()
        self._finalised = True

    ##  Merges this operation with another GroupOperation.
    #
    #   This prevents the user from having to undo multiple operations if they
    #   were not his operations.
    #
    #   The older operation must have the same number of child operations, and
    #   each pair of operations must succesfully merge, or the merge of the
    #   groupOperation will fail.
    #
    #   \param other The older GroupOperation to merge this with.
    #   \return A combination of the two group operations, or False if the operations
    #           can not be merged.
    def mergeWith(self, other):
        if type(other) is not GroupedOperation:
            return False
        if len(other._children) != len(self._children): #Must be operations on the same number of children.
            return False

        op = GroupedOperation()
        for (op1, op2) in zip(self._children, other._children):
            child_op = op1.mergeWith(op2)
            if not child_op:
                return False
            op.addOperation(child_op)
        return op
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
#==============================================================================
# libaries initialisation
#==============================================================================

"""
import os

from supsisim.const import viewTypes
#from supsisim.dialg import error

libs = dict()  # key = library name (minus the 'library_' prefix)
#                value = a set containing all blocknames

if os.path.isfile('libraries'): # home of all libaries
    libroot =  'libraries'
else:
    libroot = os.path.dirname(__file__) # home of all libaries

libprefix = 'library_'

def libpath(libname, root=libroot):
    '''return full path of libary libname'''
    if libname.startswith(libprefix):
        return os.path.join(root, libname)
    else:
        return os.path.join(root, libprefix + libname)

def blockpath(libname, blockname):
    '''return full path of libary libname'''
    return os.path.join(libpath(libname), blockname+'.py')

def moduleName(libname, blockname):
    return '{}.{}.{}'.format(os.path.basename(libroot), libprefix+libname, blockname)

def pathToLibnameBlockname(filename):
    libname, blockname = os.path.split(filename)

    if libname.startswith(libroot): # make relative if in library path
        libname = libname[len(libroot):].lstrip(os.path.sep)

    if libname.startswith(libprefix): # strip if has library_ prefix
        libname = libname[len(libprefix):].lstrip(os.path.sep)
    else:
        libname = '' # not a library element, probably toplevel diagram

    blockname, ext = os.path.splitext(blockname)
    return libname, blockname

def readLibraries():
    dirs = next(os.walk(libroot))[1]
    for dirname in dirs:
        if dirname.startswith(libprefix):
            libname = dirname[len(libprefix):]
            libs[libname] = readLibrary(libname)

def readLibrary(libname):
    files = next(os.walk(libpath(libname)))[2]
    blocks = set()
    for blockname in files:
        if not blockname.endswith('.py'): # only python files
            continue
        if blockname.startswith('__init__'): # skip __init__
            continue
        for  viewtype, (editor, extension) in viewTypes.items():
            if blockname.endswith(extension):
                if viewtype == 'blk_source':
                    blockname = blockname[:-len(extension)]
                    blocks.add(blockname)
                else: # other python type (e.g. diagram/myhdl)
                    break
    return blocks

def rmBlock(libname, blockname):
    '''remove a block from library administration (and disk)'''
    # remove self (if not yet removed)
    fp = blockpath(libname, blockname)
    if os.path.isfile(fp):
        os.remove(fp)
    # update libary administration
    libs[libname].remove(blockname)

readLibraries()
default = 'common'<commit_msg>libroot can also be local 'libraries' folder<commit_after># -*- coding: utf-8 -*-
"""
#==============================================================================
# libaries initialisation
#==============================================================================

"""
import os

from supsisim.const import viewTypes
#from supsisim.dialg import error

libs = dict()  # key = library name (minus the 'library_' prefix)
#                value = a set containing all blocknames

if os.path.isfile('libraries'): # home of all libaries
    libroot =  'libraries'
else:
    libroot = os.path.dirname(__file__) # home of all libaries

libprefix = 'library_'

def libpath(libname, root=libroot):
    '''return full path of libary libname'''
    if libname.startswith(libprefix):
        return os.path.join(root, libname)
    else:
        return os.path.join(root, libprefix + libname)

def blockpath(libname, blockname):
    '''return full path of libary libname'''
    return os.path.join(libpath(libname), blockname+'.py')

def moduleName(libname, blockname):
    return '{}.{}.{}'.format(os.path.basename(libroot), libprefix+libname, blockname)

def pathToLibnameBlockname(filename):
    libname, blockname = os.path.split(filename)

    if libname.startswith(libroot): # make relative if in library path
        libname = libname[len(libroot):].lstrip(os.path.sep)

    if libname.startswith(libprefix): # strip if has library_ prefix
        libname = libname[len(libprefix):].lstrip(os.path.sep)
    else:
        libname = '' # not a library element, probably toplevel diagram

    blockname, ext = os.path.splitext(blockname)
    return libname, blockname

def readLibraries():
    dirs = next(os.walk(libroot))[1]
    for dirname in dirs:
        if dirname.startswith(libprefix):
            libname = dirname[len(libprefix):]
            libs[libname] = readLibrary(libname)

def readLibrary(libname):
    files = next(os.walk(libpath(libname)))[2]
    blocks = set()
    for blockname in files:
        if not blockname.endswith('.py'): # only python files
            continue
        if blockname.startswith('__init__'): # skip __init__
            continue
        for  viewtype, (editor, extension) in viewTypes.items():
            if blockname.endswith(extension):
                if viewtype == 'blk_source':
                    blockname = blockname[:-len(extension)]
                    blocks.add(blockname)
                else: # other python type (e.g. diagram/myhdl)
                    break
    return blocks

def rmBlock(libname, blockname):
    '''remove a block from library administration (and disk)'''
    # remove self (if not yet removed)
    fp = blockpath(libname, blockname)
    if os.path.isfile(fp):
        os.remove(fp)
    # update libary administration
    libs[libname].remove(blockname)

readLibraries()
default = 'common'<|endoftext|>