<commit_before>from Constants import *
import os, sys, argparse
import SQLConnector
import pandas, json

def get_track_completion_status(passed_courses, failed_courses, track_reqs):
    """
    get_track_completion_status: checks if a student has completed a certain
        track, has not completed a track, or has not completed a track due to a
        failed course

    Args:
        passed_courses (list of strings): a student's passed courses
        failed_courses (list of strings): a student's failed courses
        track_reqs (list of strings): list of required courses to complete a
            certain track

    Returns:
        0 (COMPLETED) if the student completed the track
        1 (NOT_COMPLETED) if the student has yet to complete the track
        2 (HAS_FAILED) if the student has not completed the track due to a failed course
    """

    status = True

    for semester in track_reqs:
        fufilled = False

        for i in range(0, len(passed_courses)):
            if (passed_courses[i] in semester):
                passed_courses.pop(i)
                fufilled = True
                break

        if (not fufilled): # requirement unmet
            status = False
            break

    if (status): # requirement met
        return COMPLETED

    for semester in track_reqs: # check if failed course is cause
        fufilled = False;

        for i in range(0, len(failed_courses)):
            if(failed_courses[i] in semester): # a failed course was a req
                return HAS_FAILED

    return NOT_COMPLETED

def get_excel(filepath):
    """
    Returns a Pandas datafile that contains the contents of a Microsoft Excel
    Spreadsheet

    Params:
        filepath - A string containing the path to the file

    Returns:
        A Pandas datafile
    """
    return pandas.read_excel(filepath)

def get_column_names(datafile):
    """
    Returns a list containing the column names of a Pandas datafile as Python
    strings. This replaces all spaces with _ and upper-cases all letters

    Params:
        datafile - A Pandas datafile

    Returns:
        A list of strings
    """
    return [ str(s).strip().upper().replace(" ", "_") for s in datafile.columns ]

def is_table_set_up():
    """
    Returns True if this project's MySQL table is set up, False otherwise
    """
    query = "SELECT table_name FROM information_schema.tables WHERE table_schema='%s' and table_name='%s';"
    retval = SQLConnector.execute(query % (PROJECT_DB_NAME, COURSES_TABLE_NAME,))
    courses_setup = retval != None and len(retval) == 1
    retval = SQLConnector.execute(query % (PROJECT_DB_NAME, STUDENT_TABLE_NAME))
    student_setup = retval != None and len(retval) == 1

    return courses_setup and student_setup

def create_project_table(courses_column_names, student_column_names, current_courses_column_names):
    """
    Sets up MySQL with a table with a set schema given a list of column_names.
    Does nothing if the table is already set up.

    Params:
        courses_column_names - A list of strings containing column names for the courses table
        student_column_names - A list of strings containing column names for the students table

    Returns:
        None
    """
    # Tons of format strings!
    query = "CREATE TABLE IF NOT EXISTS %s (%s);"
    schema = (("%s VARCHAR(64), " * len(courses_column_names))[:-2]) % tuple(courses_column_names)
    SQLConnector.execute(query % (COURSES_TABLE_NAME, schema))

    schema = (("%s VARCHAR(64), " * len(student_column_names))[:-2]) % tuple(student_column_names)
    SQLConnector.execute(query % (STUDENT_TABLE_NAME, schema))

def delete_project_table():
    """
    Deletes the table in MySQL that this project uses. Will do nothing if the
    table does not yet exist.
    """
    query = "DROP TABLE IF EXISTS %s;"
    SQLConnector.execute(query % (COURSES_TABLE_NAME,))
    SQLConnector.execute(query % (STUDENT_TABLE_NAME,))

def reset_project_table():
    """
    Resets the table for this project by calling 'delete' and 'setup'
    """
    delete_project_table()
    for i in range(0, TOTAL_REQ_COUNT):
        STUDENT_TABLE_COLUMNS.append("REQ%02d" % i) # grad reqs
    create_project_table(COURSES_TABLE_COLUMNS, STUDENT_TABLE_COLUMNS,)

##todo change this to be load past courses
##make new func for loading new courses

def load_excel_file(datafile):
    """
    Takes the Pandas datafiles and inserts the data into the project's MySQL table.
    If the project's table is not yet created, this function will call 'create'.

    The column names are specified within COURSES_TABLE_COLUMNS and
    STUDENT_TABLE_COLUMNS

    If they are not present in the datafile, a KeyError will be raised
    """
    column_names = get_column_names(datafile) # loads file
    new_student_columns = STUDENT_TABLE_COLUMNS

    # check if COURSES_TABLE_COLUMNS are completed contained in the column names
    for name in COURSES_TABLE_COLUMNS:
        if name not in column_names:
            raise KeyError("%s is not present in the provided datafile" % name)
            sys.exit(1)

    # check if STUDENT_TABLE_COLUMNS are completely contained in the column names
    for name in new_student_columns:
        if name not in column_names:
            raise KeyError("%s is not present in the provided datafile" % name)
            sys.exit(1)

    for i in range(0, TOTAL_REQ_COUNT):
        new_student_columns.append("REQ%02d" % i) # grad reqs

    delete_project_table()
    create_project_table(COURSES_TABLE_COLUMNS, new_student_columns)

    rows, cols = datafile.shape
    counter = 0
    print "Populating %s" % COURSES_TABLE_NAME
    print "Inserting %d rows with %d data fields each" % (rows, cols)

    student_data = {} # dictionary used to temporarily store data for students datatabe

    for row in datafile.itertuples():
        if counter % 100 == 0:
            print "Progress: %d of %d" % (counter, rows)

        data = list(row._asdict().values())[1:]
        #assert(len(data) == len(column_names))

        # FIXME THIS IS BAD BUT IDK HOW TO MAKE IT BETTER - YICHENG
        # Works for now tho
        if str(data[0]) not in student_data.keys():
            student_data[str(data[0])] = [data[1], data[2], data[3], data[4]]

        data = [data[0], data[5], data[6], data[7], data[8], data[9], data[11], data[12]]

        # Generate query
        query = "INSERT INTO %s (%s) VALUES (%s);"
        schema = (("%s , " * len(COURSES_TABLE_COLUMNS))[:-2]) % tuple(COURSES_TABLE_COLUMNS)
        values = (("'%s' , " * len(data))[:-2]) % tuple(( str(it).upper() for it in data))

        query = query % (COURSES_TABLE_NAME, schema, values)
        # print query
        SQLConnector.execute(query)
        counter += 1

    print "Done!"
    print "Begin analysis... Populating %s" % STUDENT_TABLE_NAME

    script_path = os.path.realpath(__file__)
    script_loc = os.path.dirname(script_path)
    reqs = json.loads(open(script_loc + '/../data/reqs.json', 'r').read())['grad_requirements']

    query = "SELECT DISTINCT STUDENTID FROM %s;" % (COURSES_TABLE_NAME)

    res = SQLConnector.execute(query)

    print "Inserting %d students" % (len(res))

    counter = 0

    for (osis, ) in res:

        if counter % 100 == 0:
            print "Progress: %d of %d" % (counter, len(res))

        query = "SELECT COURSE FROM %s WHERE STUDENTID = %s AND \
            ((MARK >= 65 AND MARK REGEXP '^[0-9]+$') OR MARK='P' OR MARK='C' OR \
            MARK='CR' OR MARK REGEXP '^[A-D]$');" % (COURSES_TABLE_NAME, osis)

        r = SQLConnector.execute(query)
        if r:
            courses_passed = [course for (course,) in r]
        else:
            courses_passed = []

        query = "SELECT COURSE FROM %s WHERE STUDENTID = %s AND \
                ((MARK < 65 AND MARK REGEXP '^[0-9]+$') OR MARK = 'F' OR \
                MARK='NC' OR MARK = 'NS');" % (COURSES_TABLE_NAME, osis)

        r = SQLConnector.execute(query)

        if r:
            courses_failed = [course for (course, ) in r]
        else:
            courses_failed = []

        req_status = []

        for requirement in reqs:
            status = HAS_FAILED # use min to compute
            for option in requirement['options']:
                status = min(status,
                        get_track_completion_status(courses_passed, courses_failed, option['course-code']))
                if status == COMPLETED:
                    break
            req_status.append(str(status))

        query = "INSERT INTO %s (%s) VALUES (%s);"
        schema = (("%s, " * len(new_student_columns))[:-2]) % tuple(new_student_columns)
        values = (("'%s', " * len(new_student_columns))[:-2]) % tuple([osis] + student_data[str(osis)] + req_status)
        query = query % (STUDENT_TABLE_NAME, schema, values)

        r = SQLConnector.execute(query)

        counter += 1

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "-c",
        "--create",
        help="Creates the table",
        action="store_true"
    )

    parser.add_argument(
        "-d",
        "--delete",
        help="Deletes the table",
        action="store_true"
    )

    parser.add_argument(
        "-r",
        "--reset",
        help="Resets the table",
        action="store_true"
    )

    parser.add_argument(
        "-l",
        "--load",
        help="Load an Excel spreadsheet into the MySQL table",
        action="append"
    )

    parser.add_argument(
        "-t",
        "--test",
        help="Prints True if the table is set up, False otherwise",
        action="store_true"
    )

    args = parser.parse_args()

    if args.create:
        create_project_table(COURSES_TABLE_COLUMNS, STUDENT_TABLE_COLUMNS)
    elif args.delete:
        delete_project_table()
    elif args.reset:
        reset_project_table()

    if args.test:
        print(is_table_set_up())

    if args.load == None:
        return
    else:
        for f in args.load:
            load_excel_file(get_excel(f))

if __name__ == "__main__":
    main()

<commit_msg>Loren broke database_setup...<commit_after>from Constants import *
import os, sys, argparse
import SQLConnector
import pandas, json

def get_track_completion_status(passed_courses, failed_courses, track_reqs):
    """
    get_track_completion_status: checks if a student has completed a certain
        track, has not completed a track, or has not completed a track due to a
        failed course

    Args:
        passed_courses (list of strings): a student's passed courses
        failed_courses (list of strings): a student's failed courses
        track_reqs (list of strings): list of required courses to complete a
            certain track

    Returns:
        0 (COMPLETED) if the student completed the track
        1 (NOT_COMPLETED) if the student has yet to complete the track
        2 (HAS_FAILED) if the student has not completed the track due to a failed course
    """

    status = True

    for semester in track_reqs:
        fufilled = False

        for i in range(0, len(passed_courses)):
            if (passed_courses[i] in semester):
                passed_courses.pop(i)
                fufilled = True
                break

        if (not fufilled): # requirement unmet
            status = False
            break

    if (status): # requirement met
        return COMPLETED

    for semester in track_reqs: # check if failed course is cause
        fufilled = False;

        for i in range(0, len(failed_courses)):
            if(failed_courses[i] in semester): # a failed course was a req
                return HAS_FAILED

    return NOT_COMPLETED

def get_excel(filepath):
    """
    Returns a Pandas datafile that contains the contents of a Microsoft Excel
    Spreadsheet

    Params:
        filepath - A string containing the path to the file

    Returns:
        A Pandas datafile
    """
    return pandas.read_excel(filepath)

def get_column_names(datafile):
    """
    Returns a list containing the column names of a Pandas datafile as Python
    strings. This replaces all spaces with _ and upper-cases all letters

    Params:
        datafile - A Pandas datafile

    Returns:
        A list of strings
    """
    return [ str(s).strip().upper().replace(" ", "_") for s in datafile.columns ]

def is_table_set_up():
    """
    Returns True if this project's MySQL table is set up, False otherwise
    """
    query = "SELECT table_name FROM information_schema.tables WHERE table_schema='%s' and table_name='%s';"
    retval = SQLConnector.execute(query % (PROJECT_DB_NAME, COURSES_TABLE_NAME,))
    courses_setup = retval != None and len(retval) == 1
    retval = SQLConnector.execute(query % (PROJECT_DB_NAME, STUDENT_TABLE_NAME))
    student_setup = retval != None and len(retval) == 1

    return courses_setup and student_setup

def create_project_table(courses_column_names, student_column_names):
    """
    Sets up MySQL with a table with a set schema given a list of column_names.
    Does nothing if the table is already set up.

    Params:
        courses_column_names - A list of strings containing column names for the courses table
        student_column_names - A list of strings containing column names for the students table

    Returns:
        None
    """
    # Tons of format strings!
    query = "CREATE TABLE IF NOT EXISTS %s (%s);"
    schema = (("%s VARCHAR(64), " * len(courses_column_names))[:-2]) % tuple(courses_column_names)
    SQLConnector.execute(query % (COURSES_TABLE_NAME, schema))

    schema = (("%s VARCHAR(64), " * len(student_column_names))[:-2]) % tuple(student_column_names)
    SQLConnector.execute(query % (STUDENT_TABLE_NAME, schema))

def delete_project_table():
    """
    Deletes the table in MySQL that this project uses. Will do nothing if the
    table does not yet exist.
    """
    query = "DROP TABLE IF EXISTS %s;"
    SQLConnector.execute(query % (COURSES_TABLE_NAME,))
    SQLConnector.execute(query % (STUDENT_TABLE_NAME,))

def reset_project_table():
    """
    Resets the table for this project by calling 'delete' and 'setup'
    """
    delete_project_table()
    for i in range(0, TOTAL_REQ_COUNT):
        STUDENT_TABLE_COLUMNS.append("REQ%02d" % i) # grad reqs
    create_project_table(COURSES_TABLE_COLUMNS, STUDENT_TABLE_COLUMNS,)

##todo change this to be load past courses
##make new func for loading new courses

def load_excel_file(datafile):
    """
    Takes the Pandas datafiles and inserts the data into the project's MySQL table.
    If the project's table is not yet created, this function will call 'create'.

    The column names are specified within COURSES_TABLE_COLUMNS and
    STUDENT_TABLE_COLUMNS

    If they are not present in the datafile, a KeyError will be raised
    """
    column_names = get_column_names(datafile) # loads file
    new_student_columns = STUDENT_TABLE_COLUMNS

    # check if COURSES_TABLE_COLUMNS are completed contained in the column names
    for name in COURSES_TABLE_COLUMNS:
        if name not in column_names:
            raise KeyError("%s is not present in the provided datafile" % name)
            sys.exit(1)

    # check if STUDENT_TABLE_COLUMNS are completely contained in the column names
    for name in new_student_columns:
        if name not in column_names:
            raise KeyError("%s is not present in the provided datafile" % name)
            sys.exit(1)

    for i in range(0, TOTAL_REQ_COUNT):
        new_student_columns.append("REQ%02d" % i) # grad reqs

    delete_project_table()
    create_project_table(COURSES_TABLE_COLUMNS, new_student_columns)

    rows, cols = datafile.shape
    counter = 0
    print "Populating %s" % COURSES_TABLE_NAME
    print "Inserting %d rows with %d data fields each" % (rows, cols)

    student_data = {} # dictionary used to temporarily store data for students datatabe

    for row in datafile.itertuples():
        if counter % 100 == 0:
            print "Progress: %d of %d" % (counter, rows)

        data = list(row._asdict().values())[1:]
        #assert(len(data) == len(column_names))

        # FIXME THIS IS BAD BUT IDK HOW TO MAKE IT BETTER - YICHENG
        # Works for now tho
        if str(data[0]) not in student_data.keys():
            student_data[str(data[0])] = [data[1], data[2], data[3], data[4]]

        data = [data[0], data[5], data[6], data[7], data[8], data[9], data[11], data[12]]

        # Generate query
        query = "INSERT INTO %s (%s) VALUES (%s);"
        schema = (("%s , " * len(COURSES_TABLE_COLUMNS))[:-2]) % tuple(COURSES_TABLE_COLUMNS)
        values = (("'%s' , " * len(data))[:-2]) % tuple(( str(it).upper() for it in data))

        query = query % (COURSES_TABLE_NAME, schema, values)
        # print query
        SQLConnector.execute(query)
        counter += 1

    print "Done!"
    print "Begin analysis... Populating %s" % STUDENT_TABLE_NAME

    script_path = os.path.realpath(__file__)
    script_loc = os.path.dirname(script_path)
    reqs = json.loads(open(script_loc + '/../data/reqs.json', 'r').read())['grad_requirements']

    query = "SELECT DISTINCT STUDENTID FROM %s;" % (COURSES_TABLE_NAME)

    res = SQLConnector.execute(query)

    print "Inserting %d students" % (len(res))

    counter = 0

    for (osis, ) in res:

        if counter % 100 == 0:
            print "Progress: %d of %d" % (counter, len(res))

        query = "SELECT COURSE FROM %s WHERE STUDENTID = %s AND \
            ((MARK >= 65 AND MARK REGEXP '^[0-9]+$') OR MARK='P' OR MARK='C' OR \
            MARK='CR' OR MARK REGEXP '^[A-D]$');" % (COURSES_TABLE_NAME, osis)

        r = SQLConnector.execute(query)
        if r:
            courses_passed = [course for (course,) in r]
        else:
            courses_passed = []

        query = "SELECT COURSE FROM %s WHERE STUDENTID = %s AND \
                ((MARK < 65 AND MARK REGEXP '^[0-9]+$') OR MARK = 'F' OR \
                MARK='NC' OR MARK = 'NS');" % (COURSES_TABLE_NAME, osis)

        r = SQLConnector.execute(query)

        if r:
            courses_failed = [course for (course, ) in r]
        else:
            courses_failed = []

        req_status = []

        for requirement in reqs:
            status = HAS_FAILED # use min to compute
            for option in requirement['options']:
                status = min(status,
                        get_track_completion_status(courses_passed, courses_failed, option['course-code']))
                if status == COMPLETED:
                    break
            req_status.append(str(status))

        query = "INSERT INTO %s (%s) VALUES (%s);"
        schema = (("%s, " * len(new_student_columns))[:-2]) % tuple(new_student_columns)
        values = (("'%s', " * len(new_student_columns))[:-2]) % tuple([osis] + student_data[str(osis)] + req_status)
        query = query % (STUDENT_TABLE_NAME, schema, values)

        r = SQLConnector.execute(query)

        counter += 1

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "-c",
        "--create",
        help="Creates the table",
        action="store_true"
    )

    parser.add_argument(
        "-d",
        "--delete",
        help="Deletes the table",
        action="store_true"
    )

    parser.add_argument(
        "-r",
        "--reset",
        help="Resets the table",
        action="store_true"
    )

    parser.add_argument(
        "-l",
        "--load",
        help="Load an Excel spreadsheet into the MySQL table",
        action="append"
    )

    parser.add_argument(
        "-t",
        "--test",
        help="Prints True if the table is set up, False otherwise",
        action="store_true"
    )

    args = parser.parse_args()

    if args.create:
        create_project_table(COURSES_TABLE_COLUMNS, STUDENT_TABLE_COLUMNS)
    elif args.delete:
        delete_project_table()
    elif args.reset:
        reset_project_table()

    if args.test:
        print(is_table_set_up())

    if args.load == None:
        return
    else:
        for f in args.load:
            load_excel_file(get_excel(f))

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>"""Implement parsers for STIX 2.0 JSON Cyber-Domain Observable objects."""
# standard library
from typing import Iterable, Union

# third-party
from stix2 import AutonomousSystem, DomainName, EmailAddress

from .threat_actor import StixModel  # pylint: disable=relative-beyond-top-level

class StixASObject(StixModel):
    """Parser for STIX AS Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070683
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[AutonomousSystem]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'AS Number'

        parse_map = {
            'type': 'autonomous-system',
            'spec_version': '2.1',
            'id': '@.id',
            'number': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield AutonomousSystem(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        for data in list(stix_data):
            yield self._map(data, {'hostName': '@.value'})

class StixDomainNameObject(StixModel):
    """Parser for STIX Domain Name Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070687
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[DomainName]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'hostName'

        parse_map = {
            'type': 'domain-name',
            'spec_version': '2.1',
            'id': '@.id',
            'value': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield DomainName(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        for data in list(stix_data):
            yield self._map(data, {'hostName': '@.value'})

class StixEmailAddressObject(StixModel):
    """Parser for STIX Email Address Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070690
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[EmailAddress]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'address'

        parse_map = {
            'type': 'email-addr',
            'spec_version': '2.1',
            'id': '@.id',
            'value': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield EmailAddress(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        for data in list(stix_data):
            yield self._map(data, {'hostName': '@.value'})
<commit_msg>add consume methods<commit_after>"""Implement parsers for STIX 2.0 JSON Cyber-Domain Observable objects."""
# standard library
from typing import Iterable, Union

# third-party
from stix2 import AutonomousSystem, DomainName, EmailAddress

from .threat_actor import StixModel  # pylint: disable=relative-beyond-top-level

class StixASObject(StixModel):
    """Parser for STIX AS Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070683
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[AutonomousSystem]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'AS Number'

        parse_map = {
            'type': 'autonomous-system',
            'spec_version': '2.1',
            'id': '@.id',
            'number': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield AutonomousSystem(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        parse_map = {'type': 'ASN', 'summary': '@.value', 'external_id': '@.id'}

        for data in self._map(stix_data, parse_map):
            yield {
                'summary': data.get('summary'),
                'type': data.get('type'),
                'attributes': [{'type': 'External ID', 'value': data.get('external_id')}],
            }

class StixDomainNameObject(StixModel):
    """Parser for STIX Domain Name Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070687
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[DomainName]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'hostName'

        parse_map = {
            'type': 'domain-name',
            'spec_version': '2.1',
            'id': '@.id',
            'value': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield DomainName(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        parse_map = {'type': 'Host', 'summary': '@.value', 'external_id': '@.id'}

        for data in self._map(stix_data, parse_map):
            yield {
                'summary': data.get('summary'),
                'type': data.get('type'),
                'attributes': [{'type': 'External ID', 'value': data.get('external_id')}],
            }

class StixEmailAddressObject(StixModel):
    """Parser for STIX Email Address Object.

    see: https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070690
    """

    def produce(self, tc_data: Union[list, dict]) -> Iterable[EmailAddress]:
        """Produce STIX 2.0 JSON object from TC API response."""
        if isinstance(tc_data, list) and len(tc_data) > 0 and 'summary' in tc_data[0]:
            indicator_field = 'summary'
        else:
            indicator_field = 'address'

        parse_map = {
            'type': 'email-addr',
            'spec_version': '2.1',
            'id': '@.id',
            'value': f'@."{indicator_field}"',
        }

        for stix_data in self._map(tc_data, parse_map):
            yield EmailAddress(**stix_data)

    def consume(self, stix_data: Union[list, dict]):
        """Produce a ThreatConnect object from a STIX 2.0 JSON object."""
        parse_map = {'type': 'EmailAddress', 'summary': '@.value', 'external_id': '@.id'}

        for data in self._map(stix_data, parse_map):
            yield {
                'summary': data.get('summary'),
                'type': data.get('type'),
                'attributes': [{'type': 'External ID', 'value': data.get('external_id')}],
            }
<|endoftext|>


<commit_before>#! /usr/bin/python2
# -*- coding: utf-8 -*-
#
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Merges several pronunciation dictionaries.

This is a priority-union in which an entry read from a lexicon later on the
command line superseded any entry with an identical key read earlier.

Can be used to patch lexicons by merging a large base dictionary with a small
list of corrections.

"""

from __future__ import unicode_literals

import codecs
import sys

STDIN = codecs.getreader('utf-8')(sys.stdin)
STDOUT = codecs.getwriter('utf-8')(sys.stdout)
STDERR = codecs.getwriter('utf-8')(sys.stderr)

def ReadTsvLexicon(path):
  """Reads a lexicon in TSV format. Only the first 2 columns are used."""
  lex = {}
  with codecs.open(path, 'r', 'utf-8') as reader:
    for line in reader:
      line = line.rstrip('\n')
      fields = line.split('\t')
      assert len(fields) >= 2
      orth, pron = fields[:2]
      if orth not in lex:
        lex[orth] = []
      lex[orth].append(pron)
  return lex

def main(argv):
  if len(argv) == 1:
    STDOUT.write('Usage: %s LEXICON...\n' % argv[0])
    sys.exit(1)
  lex = {}
  for path in argv[1:]:
    lex.update(ReadTsvLexicon(path))
  words = lex.keys()
  words.sort()
  for orth in words:
    for pron in lex[orth]:
      STDOUT.write('%s\t%s\n' % (orth, pron))
  return

if __name__ == '__main__':
  main(sys.argv)
<commit_msg>Allow merge_lexicons input to come from stdin.<commit_after>#! /usr/bin/python2
# -*- coding: utf-8 -*-
#
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Merges several pronunciation dictionaries.

This is a priority-union in which an entry read from a lexicon later on the
command line superseded any entry with an identical key read earlier.

Can be used to patch lexicons by merging a large base dictionary with a small
list of corrections.

"""

from __future__ import unicode_literals

import codecs
import sys

STDIN = codecs.getreader('utf-8')(sys.stdin)
STDOUT = codecs.getwriter('utf-8')(sys.stdout)
STDERR = codecs.getwriter('utf-8')(sys.stderr)

def ReadTsvLexicon(reader):
  """Reads a lexicon in TSV format. Only the first 2 columns are used."""
  lex = {}
  for line in reader:
    line = line.rstrip('\n')
    fields = line.split('\t')
    assert len(fields) >= 2
    orth, pron = fields[:2]
    if orth not in lex:
      lex[orth] = []
    lex[orth].append(pron)
  return lex

def main(argv):
  if len(argv) == 1:
    STDOUT.write('Usage: %s LEXICON...\n' % argv[0])
    sys.exit(2)
  lex = {}
  for path in argv[1:]:
    if path == '-':
      lex.update(ReadTsvLexicon(STDIN))
    else:
      with codecs.open(path, 'r', 'utf-8') as reader:
        lex.update(ReadTsvLexicon(reader))
  words = lex.keys()
  words.sort()
  for orth in words:
    for pron in lex[orth]:
      STDOUT.write('%s\t%s\n' % (orth, pron))
  return

if __name__ == '__main__':
  main(sys.argv)
<|endoftext|>


<commit_before>import re

from django.db import models
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.test import TestCase

import LabtrackerCore.utils as utils
import LabtrackerCore.models as coreModels

from datetime import date

from south.modelsinspector import add_introspection_rules

add_introspection_rules([], ["Machine.models.MacField"])
class MacField(models.CharField):

    description = "A field for mac addresses"
    default_error_messages = {
        'invalid': (u'Enter a valid MAC address.'),
    }

    def __init__(self, *args, **kwargs):
        kwargs['max_length'] = 17
        kwargs['help_text'] = 'Please use the following format: 00:AA:1B:00:00:00.'
        super(MacField, self).__init__(*args, **kwargs)

    def db_type(self, connection):
        return 'char(17)'

    def validate(self, value, model_instance):
        if re.search('^([0-9a-fA-F]{2}([:]?|$)){6}$', value) == None:
            raise ValidationError('Enter a valid MAC address.')

class Status(models.Model):
    """
    Status of the machine
    """

    ms_id = models.AutoField(primary_key=True)
    name = models.SlugField(max_length=60, unique=True)
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name_plural="Status"
        unique_together = (("ms_id", "name"),)

class Platform(models.Model):
    """
    Machine Platform: windows XP, Vista, Mac OS X, Linux/Ubuntu, etc.
    """

    platform_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

class Type(models.Model):
    """
    Type of Machine, not InventoryType, similar to a group
    """

    mt_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    platform = models.ForeignKey(Platform)
    model_name = models.CharField(max_length=60)
    specs = models.TextField()
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

class Location(models.Model):
    ml_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    building = models.CharField(max_length=60, null=True)
    floor = models.SmallIntegerField(null=True, blank=True)
    room = models.CharField(max_length=30, null=True)
    comment = models.CharField(max_length=600)
    usable_threshold = models.IntegerField(default=95)

    def __unicode__(self):
        return self.name

class Item(coreModels.Item):
    """
    The Machine
    """
    core = models.OneToOneField(coreModels.Item, parent_link=True,
            editable=False)
    type = models.ForeignKey(Type, verbose_name='Machine Type')
    verified = models.BooleanField()
    unusable = models.BooleanField()
    retired = models.BooleanField()
    status = models.ManyToManyField(Status, related_name="machine_status")

    location = models.ForeignKey(Location, verbose_name='Location')
    ip = models.IPAddressField(verbose_name="IP Address")
    mac1 = MacField(verbose_name='MAC Address')
    mac2 = MacField(verbose_name='Additional MAC Address', blank=True)
    mac3 = MacField(verbose_name='Additional MAC Address', blank=True)
    wall_port = models.CharField(max_length=25)
    date_added = models.DateTimeField(auto_now_add=True)
    last_modified = models.DateTimeField(auto_now=True)

    manu_tag = models.CharField(max_length=200,
            verbose_name="Manufacturers tag")
    uw_tag = models.CharField(max_length=200, verbose_name="UW tag",
            blank=True, null=True)

    purchase_date = models.DateField(null=True, blank=True)
    warranty_date = models.DateField(null=True, blank=True)
    stf_date = models.DateField(null=True, blank=True,
            verbose_name='Student Tech Fee Contract Expiration')

    comment = models.TextField(blank=True, null=True)

    def usable(self):
        return (not self.unusable)
    usable.boolean = True

    @models.permalink
    def get_absolute_url(self):
        return ('Machine-detail', [str(self.pk),])

    def primaryContact(self):
        """
        Returns the primary contact for this item, or None if no primary
        contact exists
        """
        # first check to see if item has groups, then fetch the groups, and
        # call the groups primary contact function
        return Contact.objects.filter(is_primary=True, mg__items__pk=self.pk)

    def __unicode__(self):
        return self.item.name

    def delete(self):
        self.core.delete()          # delete the item in coreModels.Item
        super(Item,self).delete()   # delete self

    def save(self, *args, **kwargs):
        self.it = utils.getInventoryType(__name__)
        super(Item,self).save(*args, **kwargs)

class Group(coreModels.Group):
    """
    Expands on the coreModels.Group
    """
    core = models.OneToOneField(coreModels.Group, parent_link=True, editable=False)
    is_lab = models.BooleanField()
    casting_server = models.IPAddressField()
    gateway = models.IPAddressField()

    def primaryContact(self):
        """
        for this group find primary contact, returns set of users
        """
        return self.contact_set.filter(is_primary=True)

    def contacts(self):
        """
        Returns the set of all contacts for this group
        """
        return self.contact_set.filter()

    def __unicode__(self):
        return self.group.name

    def delete(self):
        self.group.delete()
        super(Group,self).delete()

    def save(self):
        if (self.it == None):
            self.it = utils.getInventoryType(__name__)
        super(Group,self).save()

class History(models.Model):
    mh_id = models.AutoField(primary_key=True)
    machine = models.ForeignKey(Item)
    ms = models.ManyToManyField(Status, null=True, blank=True)
    user = models.ForeignKey(coreModels.LabUser)
    session_time = models.DecimalField(max_digits=16, decimal_places=2, null=True, blank=True)
    login_time = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
	return "%s-%s-%s" % (self.machine.location,self.machine.name,self.login_time)

class Contact(models.Model):
    contact_id = models.AutoField(primary_key=True)
    mg = models.ForeignKey(Group)

    user = models.ForeignKey(User)
    is_primary = models.BooleanField(default=False)

    def __unicode__(self):
        extra = ("", " (Primary)")[self.is_primary]
        return "%s - %s%s" % (self.mg, self.user, extra)

"""
Test Cases
"""
class PlatformTest(TestCase):
    def setUp(self):
        """
        Create a platform
        """
        self.name = "ArchLinux"
        self.platform = Platform.objects.create(
                    name = "ArchLinux",
                    description = "ArchLinux distribution"
                )

    def testExistance(self):
        platforms = Platform.objects.filter(name="ArchLinux")
        assert(len(platforms) == 1)
        self.assertEquals(platforms[0], self.platform)

    def tearDown(self):
        self.platform.delete()

class TypeTest(TestCase):
    def setUp(self):
        """
        Create a test item
        """
        self.name = "Gaming Station"
        self.type = Type.objects.create(
                name = "Gaming Station",
                model_name = "Dell Optiplex 2002",
                platform = Platform.objects.all()[0],
                specs = "stuff",
                description = ""
            )

    def tearDown(self):
        self.type.delete()

class StatusTest(TestCase):
    def setUp(self):
        self.name = "test"
        self.status = Status.objects.create(
                description = "teststatus",
                name = "test",
            )
        self.status.save()

    def testExistance(self):
        status = Status.objects.filter(name=self.name)
        assert(len(status) == 1)
        self.assertEquals(status[0], self.status)

class LocationTest(TestCase):
    def setUp(self):
        self.name = "test"
        self.location = Location.objects.create(
                building = "test",
                comment = "test",
                room = 101,
                name = self.name,
                floor = 1
            )
        self.location.save()

    def testExistance(self):
        location = Location.objects.filter(name=self.name)
        assert(len(location) == 1)
        self.assertEquals(location[0], self.location)

class MachineItemTest(TestCase):
    def setUp(self):
        """
        A create a machine item
        """

        coreModels.InventoryType.objects.create(
                    name="Machine",
                    namespace="Machine",
                    description="Machine inv type"
                )

        self.platform = Platform.objects.create(
                    name = "ArchLinux",
                    description = "ArchLinux distribution"
                )
        self.platform.save()

        self.type = Type.objects.create(
                name = "Gaming Station",
                model_name = "Dell Optiplex 2002",
                platform = Platform.objects.all()[0],
                specs = "stuff",
                description = ""
            )
        self.type.save()

        self.status = Status.objects.create(
                description = "teststatus",
                name = "test",
            )
        self.status.save()

        self.location = Location.objects.create(
                building = "test",
                comment = "test",
                room = 101,
                name = "test",
                floor = 1
            )
        self.location.save()

        self.name = "test_name_abcd"

        self.item = Item.objects.create(
                name=self.name,
                type=self.type,
                location=self.location,
                purchase_date = "2009-01-07",
                warranty_date = "2009-01-07",
                wall_port = "adj",
                ip="18.121.342.11",
                mac1="00:34:A3:DF:XA:90",
                manu_tag="manufactuer tag",
                uw_tag="uw tag",
                comment="comment")
        self.item.status.add(self.status)
        self.item.save()

    def testParent(self):
        """
        Test and see if the parent exists
        """
        parent = coreModels.Item.objects.get(name=self.name)
        self.assertEquals(parent.item.name, self.name)
<commit_msg>fix #394 allow the Item Status field to have blank=true<commit_after>import re

from django.db import models
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.test import TestCase

import LabtrackerCore.utils as utils
import LabtrackerCore.models as coreModels

from datetime import date

from south.modelsinspector import add_introspection_rules

add_introspection_rules([], ["Machine.models.MacField"])
class MacField(models.CharField):

    description = "A field for mac addresses"
    default_error_messages = {
        'invalid': (u'Enter a valid MAC address.'),
    }

    def __init__(self, *args, **kwargs):
        kwargs['max_length'] = 17
        kwargs['help_text'] = 'Please use the following format: 00:AA:1B:00:00:00.'
        super(MacField, self).__init__(*args, **kwargs)

    def db_type(self, connection):
        return 'char(17)'

    def validate(self, value, model_instance):
        if re.search('^([0-9a-fA-F]{2}([:]?|$)){6}$', value) == None:
            raise ValidationError('Enter a valid MAC address.')

class Status(models.Model):
    """
    Status of the machine
    """

    ms_id = models.AutoField(primary_key=True)
    name = models.SlugField(max_length=60, unique=True)
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name_plural="Status"
        unique_together = (("ms_id", "name"),)

class Platform(models.Model):
    """
    Machine Platform: windows XP, Vista, Mac OS X, Linux/Ubuntu, etc.
    """

    platform_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

class Type(models.Model):
    """
    Type of Machine, not InventoryType, similar to a group
    """

    mt_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    platform = models.ForeignKey(Platform)
    model_name = models.CharField(max_length=60)
    specs = models.TextField()
    description = models.CharField(max_length=400, blank=True)

    def __unicode__(self):
        return self.name

class Location(models.Model):
    ml_id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=60, unique=True)
    building = models.CharField(max_length=60, null=True)
    floor = models.SmallIntegerField(null=True, blank=True)
    room = models.CharField(max_length=30, null=True)
    comment = models.CharField(max_length=600)
    usable_threshold = models.IntegerField(default=95)

    def __unicode__(self):
        return self.name

class Item(coreModels.Item):
    """
    The Machine
    """
    core = models.OneToOneField(coreModels.Item, parent_link=True,
            editable=False)
    type = models.ForeignKey(Type, verbose_name='Machine Type')
    verified = models.BooleanField()
    unusable = models.BooleanField()
    retired = models.BooleanField()
    status = models.ManyToManyField(Status, related_name="machine_status", blank=True)

    location = models.ForeignKey(Location, verbose_name='Location')
    ip = models.IPAddressField(verbose_name="IP Address")
    mac1 = MacField(verbose_name='MAC Address')
    mac2 = MacField(verbose_name='Additional MAC Address', blank=True)
    mac3 = MacField(verbose_name='Additional MAC Address', blank=True)
    wall_port = models.CharField(max_length=25)
    date_added = models.DateTimeField(auto_now_add=True)
    last_modified = models.DateTimeField(auto_now=True)

    manu_tag = models.CharField(max_length=200,
            verbose_name="Manufacturers tag")
    uw_tag = models.CharField(max_length=200, verbose_name="UW tag",
            blank=True, null=True)

    purchase_date = models.DateField(null=True, blank=True)
    warranty_date = models.DateField(null=True, blank=True)
    stf_date = models.DateField(null=True, blank=True,
            verbose_name='Student Tech Fee Contract Expiration')

    comment = models.TextField(blank=True, null=True)

    def usable(self):
        return (not self.unusable)
    usable.boolean = True

    @models.permalink
    def get_absolute_url(self):
        return ('Machine-detail', [str(self.pk),])

    def primaryContact(self):
        """
        Returns the primary contact for this item, or None if no primary
        contact exists
        """
        # first check to see if item has groups, then fetch the groups, and
        # call the groups primary contact function
        return Contact.objects.filter(is_primary=True, mg__items__pk=self.pk)

    def __unicode__(self):
        return self.item.name

    def delete(self):
        self.core.delete()          # delete the item in coreModels.Item
        super(Item,self).delete()   # delete self

    def save(self, *args, **kwargs):
        self.it = utils.getInventoryType(__name__)
        super(Item,self).save(*args, **kwargs)

class Group(coreModels.Group):
    """
    Expands on the coreModels.Group
    """
    core = models.OneToOneField(coreModels.Group, parent_link=True, editable=False)
    is_lab = models.BooleanField()
    casting_server = models.IPAddressField()
    gateway = models.IPAddressField()

    def primaryContact(self):
        """
        for this group find primary contact, returns set of users
        """
        return self.contact_set.filter(is_primary=True)

    def contacts(self):
        """
        Returns the set of all contacts for this group
        """
        return self.contact_set.filter()

    def __unicode__(self):
        return self.group.name

    def delete(self):
        self.group.delete()
        super(Group,self).delete()

    def save(self):
        if (self.it == None):
            self.it = utils.getInventoryType(__name__)
        super(Group,self).save()

class History(models.Model):
    mh_id = models.AutoField(primary_key=True)
    machine = models.ForeignKey(Item)
    ms = models.ManyToManyField(Status, null=True, blank=True)
    user = models.ForeignKey(coreModels.LabUser)
    session_time = models.DecimalField(max_digits=16, decimal_places=2, null=True, blank=True)
    login_time = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
	return "%s-%s-%s" % (self.machine.location,self.machine.name,self.login_time)

class Contact(models.Model):
    contact_id = models.AutoField(primary_key=True)
    mg = models.ForeignKey(Group)

    user = models.ForeignKey(User)
    is_primary = models.BooleanField(default=False)

    def __unicode__(self):
        extra = ("", " (Primary)")[self.is_primary]
        return "%s - %s%s" % (self.mg, self.user, extra)

"""
Test Cases
"""
class PlatformTest(TestCase):
    def setUp(self):
        """
        Create a platform
        """
        self.name = "ArchLinux"
        self.platform = Platform.objects.create(
                    name = "ArchLinux",
                    description = "ArchLinux distribution"
                )

    def testExistance(self):
        platforms = Platform.objects.filter(name="ArchLinux")
        assert(len(platforms) == 1)
        self.assertEquals(platforms[0], self.platform)

    def tearDown(self):
        self.platform.delete()

class TypeTest(TestCase):
    def setUp(self):
        """
        Create a test item
        """
        self.name = "Gaming Station"
        self.type = Type.objects.create(
                name = "Gaming Station",
                model_name = "Dell Optiplex 2002",
                platform = Platform.objects.all()[0],
                specs = "stuff",
                description = ""
            )

    def tearDown(self):
        self.type.delete()

class StatusTest(TestCase):
    def setUp(self):
        self.name = "test"
        self.status = Status.objects.create(
                description = "teststatus",
                name = "test",
            )
        self.status.save()

    def testExistance(self):
        status = Status.objects.filter(name=self.name)
        assert(len(status) == 1)
        self.assertEquals(status[0], self.status)

class LocationTest(TestCase):
    def setUp(self):
        self.name = "test"
        self.location = Location.objects.create(
                building = "test",
                comment = "test",
                room = 101,
                name = self.name,
                floor = 1
            )
        self.location.save()

    def testExistance(self):
        location = Location.objects.filter(name=self.name)
        assert(len(location) == 1)
        self.assertEquals(location[0], self.location)

class MachineItemTest(TestCase):
    def setUp(self):
        """
        A create a machine item
        """

        coreModels.InventoryType.objects.create(
                    name="Machine",
                    namespace="Machine",
                    description="Machine inv type"
                )

        self.platform = Platform.objects.create(
                    name = "ArchLinux",
                    description = "ArchLinux distribution"
                )
        self.platform.save()

        self.type = Type.objects.create(
                name = "Gaming Station",
                model_name = "Dell Optiplex 2002",
                platform = Platform.objects.all()[0],
                specs = "stuff",
                description = ""
            )
        self.type.save()

        self.status = Status.objects.create(
                description = "teststatus",
                name = "test",
            )
        self.status.save()

        self.location = Location.objects.create(
                building = "test",
                comment = "test",
                room = 101,
                name = "test",
                floor = 1
            )
        self.location.save()

        self.name = "test_name_abcd"

        self.item = Item.objects.create(
                name=self.name,
                type=self.type,
                location=self.location,
                purchase_date = "2009-01-07",
                warranty_date = "2009-01-07",
                wall_port = "adj",
                ip="18.121.342.11",
                mac1="00:34:A3:DF:XA:90",
                manu_tag="manufactuer tag",
                uw_tag="uw tag",
                comment="comment")
        self.item.status.add(self.status)
        self.item.save()

    def testParent(self):
        """
        Test and see if the parent exists
        """
        parent = coreModels.Item.objects.get(name=self.name)
        self.assertEquals(parent.item.name, self.name)
<|endoftext|>


<commit_before>#!/bin/python

import os, sys, time

lines = open(sys.argv[1], 'rb').readlines()

# logfile format:
# <time(ms)> <key>: <value>
# example:
# 16434 read cache: 17

keys = ['read', 'write', 'head movement', 'seek per read byte', 'seek per written byte']
colors = ['70e070', 'e07070', '3030f0', '10a010', 'a01010']
style = ['linespoints', 'points pointtype 2', 'lines', 'lines', 'lines']
axis = ['x1y1', 'x1y1', 'x1y2', 'x1y2', 'x1y2']
plot = [True, False, False, False, False]

out = open('disk_access_log.dat', 'w+')

time = 1000

last_pos = 0
last_t = 0
cur_movement = 0
cur_read = 0
cur_write = 0

for l in lines:
	try:
		# strip newline
		l = l[0:-1].split(' ')
		t = int(l[0])
		k = l[1]
		n = int(l[2])
	except:
		print l
		continue

	read = '-'
	write = '-'
	movement = '-'
	amount_read = '-'
	amount_write = '-'
	if k == 'read': read = '%d' % n
	if k == 'write': write = '%d' % n
	if k == 'read_end': cur_read += n - last_pos
	if k == 'write_end': cur_write += n - last_pos

	cur_movement += abs(last_pos - n)
	last_pos = n

	if last_t + time <= t:
		movement = '%d' % cur_movement
		if cur_read > 0:
			amount_read = '%d' % (cur_movement / cur_read)
		if cur_write > 0:
			amount_write = '%d' % (cur_movement / cur_write)
		cur_movement = 0
		cur_read = 0
		cur_write = 0
		last_t = t

	print >>out, '%d\t%s\t%s\t%s\t%s\t%s' % (t, read, write, movement, amount_read, amount_write)

out.close()

out = open('disk_access.gnuplot', 'wb')
print >>out, "set term png size 600,300"
print >>out, 'set output "disk_access.png"'
print >>out, 'set xrange [*:*]'
#print >>out, 'set y2range [0:*]'
print >>out, 'set xlabel "time (ms)"'
print >>out, 'set ylabel "file position"'
#print >>out, 'set y2label "bytes / %d second(s)"' % (time / 1000)
print >>out, "set key box"
print >>out, "set tics nomirror"
#print >>out, "set y2tics 100"
print >>out, 'plot',
count = 1
for k in keys:
	count += 1
	if not plot[count-2]: continue
	print >>out, ' "disk_access_log.dat" using 1:%d title "%s" with %s lt rgb "#%s" axis %s,' \
		% (count, k, style[count-2], colors[count-2], axis[count-2]),
print >>out, 'x=0'
out.close()

os.system('gnuplot disk_access.gnuplot')

<commit_msg>restored disk_access graph size<commit_after>#!/bin/python

import os, sys, time

lines = open(sys.argv[1], 'rb').readlines()

# logfile format:
# <time(ms)> <key>: <value>
# example:
# 16434 read cache: 17

keys = ['read', 'write', 'head movement', 'seek per read byte', 'seek per written byte']
colors = ['70e070', 'e07070', '3030f0', '10a010', 'a01010']
style = ['linespoints', 'points pointtype 2', 'lines', 'lines', 'lines']
axis = ['x1y1', 'x1y1', 'x1y2', 'x1y2', 'x1y2']
plot = [True, False, False, False, False]

out = open('disk_access_log.dat', 'w+')

time = 1000

last_pos = 0
last_t = 0
cur_movement = 0
cur_read = 0
cur_write = 0

for l in lines:
	try:
		# strip newline
		l = l[0:-1].split(' ')
		t = int(l[0])
		k = l[1]
		n = int(l[2])
	except:
		print l
		continue

	read = '-'
	write = '-'
	movement = '-'
	amount_read = '-'
	amount_write = '-'
	if k == 'read': read = '%d' % n
	if k == 'write': write = '%d' % n
	if k == 'read_end': cur_read += n - last_pos
	if k == 'write_end': cur_write += n - last_pos

	cur_movement += abs(last_pos - n)
	last_pos = n

	if last_t + time <= t:
		movement = '%d' % cur_movement
		if cur_read > 0:
			amount_read = '%d' % (cur_movement / cur_read)
		if cur_write > 0:
			amount_write = '%d' % (cur_movement / cur_write)
		cur_movement = 0
		cur_read = 0
		cur_write = 0
		last_t = t

	print >>out, '%d\t%s\t%s\t%s\t%s\t%s' % (t, read, write, movement, amount_read, amount_write)

out.close()

out = open('disk_access.gnuplot', 'wb')
print >>out, "set term png size 1200,900"
print >>out, 'set output "disk_access.png"'
print >>out, 'set xrange [*:*]'
#print >>out, 'set y2range [0:*]'
print >>out, 'set xlabel "time (ms)"'
print >>out, 'set ylabel "file position"'
#print >>out, 'set y2label "bytes / %d second(s)"' % (time / 1000)
print >>out, "set key box"
print >>out, "set tics nomirror"
#print >>out, "set y2tics 100"
print >>out, 'plot',
count = 1
for k in keys:
	count += 1
	if not plot[count-2]: continue
	print >>out, ' "disk_access_log.dat" using 1:%d title "%s" with %s lt rgb "#%s" axis %s,' \
		% (count, k, style[count-2], colors[count-2], axis[count-2]),
print >>out, 'x=0'
out.close()

os.system('gnuplot disk_access.gnuplot')

<|endoftext|>


<commit_before><commit_msg>strips possible schema/table info from a column name returned in cursor metadata<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: sts=4 sw=4 et
# GladeVcp actions
#
# Copyright (c) 2011  Pavel Shramov <shramov@mexmat.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

import os, time

import gobject, gtk

from hal_widgets import _HalWidgetBase
import linuxcnc
from hal_glib import GStat
from hal_actions import _EMC_ActionBase, _EMC_Action
from hal_filechooser import _EMC_FileChooser

import gtksourceview2 as gtksourceview

class EMC_SourceView(gtksourceview.View, _EMC_ActionBase):
    __gtype_name__ = 'EMC_SourceView'
    def __init__(self, *a, **kw):
        gtksourceview.View.__init__(self, *a, **kw)
        self.filename = None
        self.mark = None
        self.offset = 0
        self.program_length = 0
        self.buf = gtksourceview.Buffer()
        self.buf.set_max_undo_levels(20)
        self.buf.connect('changed', self.update_iter)
        self.set_buffer(self.buf)
        self.lm = gtksourceview.LanguageManager()
        self.sm = gtksourceview.StyleSchemeManager()
        if 'EMC2_HOME' in os.environ:
            path = os.path.join(os.environ['EMC2_HOME'], 'share/gtksourceview-2.0/language-specs/')
            self.lm.set_search_path(self.lm.get_search_path() + [path])

        self.buf.set_language(self.lm.get_language('.ngc'))
        self.set_show_line_numbers(True)
        self.set_show_line_marks(True)
        self.set_highlight_current_line(True)
        self.set_mark_category_icon_from_icon_name('motion', 'gtk-forward')
        self.set_mark_category_background('motion', gtk.gdk.Color('#ff0'))
        self.found_text_tag = self.buf.create_tag(background="yellow")
        self.update_iter()
        self.connect('button-release-event', self.button_pressed)

    def _hal_init(self):
        _EMC_ActionBase._hal_init(self)
        self.gstat.connect('file-loaded', lambda w, f: gobject.timeout_add(1, self.load_file, f))
        self.gstat.connect('line-changed', self.highlight_line)
        self.gstat.connect('interp_idle', lambda w: self.set_line_number(0))

    def set_language(self, lang, path = None):
        # path = the search path for the langauage file
        # if none, set to default
        # lang = the lang file to set
        if path == None:
            if 'EMC2_HOME' in os.environ:
                path = os.path.join(os.environ['EMC2_HOME'], 'share/gtksourceview-2.0/language-specs/')
        if path:
            self.lm.set_search_path(path)
        self.buf.set_language(self.lm.get_language(lang))

    def set_style_scheme(self, style, path = None):
        if path:
            self.sm.set_search_path(path)
        self.buf.set_style_scheme(self.sm.get_scheme(style))

    def get_filename(self):
        return self.filename

    # This load the file while not allowing undo buttons to unload the program.
    # It updates the iter because iters become invalid when anything changes.
    # We set the buffer-unmodified flag false after loading the file.
    # Set the hilight line to the line linuxcnc is looking at.
    # if one calls load_file without a filenname, We reload the exisiting file.
    def load_file(self, fn=None):
        self.buf.begin_not_undoable_action()
        if fn == None:
            fn = self.filename
        self.filename = fn
        if not fn:
            self.buf.set_text('')
            return
        self.buf.set_text(open(fn).read())
        self.buf.end_not_undoable_action()
        self.buf.set_modified(False)
        self.update_iter()
        self.highlight_line(self.gstat, self.gstat.stat.motion_line)
        self.offset = self.gstat.stat.motion_line
        f = file(fn, 'r')
        p = f.readlines()
        f.close()
        self.program_length = len(p)

    # This moves the highlight line to a lower numbered line.
    # useful for run-at-line selection
    def line_down(self):
        self.offset +=1
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    # This moves the highlight line to a higher numbered line.
    # useful for run-at-line selection
    def line_up(self):
        self.offset -=1
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    def get_line_number(self):
        return self.offset

    # sets the highlight line to a specified line.
    def set_line_number(self,linenum):
        self.offset = linenum
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    def check_offset(self):
        if self.offset < 0:
            self.offset = 0
        elif  self.offset > self.program_length:
            self.offset = self.program_length

    def highlight_line(self, w, l):
        self.offset = l
        if not l:
            if self.mark:
                self.buf.delete_mark(self.mark)
                self.mark = None
            return
        line = self.buf.get_iter_at_line(l-1)
        if not self.mark:
            self.mark = self.buf.create_source_mark('motion', 'motion', line)
            self.mark.set_visible(True)
        else:
            self.buf.move_mark(self.mark, line)
        self.scroll_to_mark(self.mark, 0, True, 0, 0.5)

    def button_pressed(self,widget,event):
        self.update_iter()

    # iters are invalid (and will cause a complete crash) after any changes.
    # so we have to update them after a change or the user clicks on view with mouse
    # re-establish start and end of text
    # current_iter is the cursor position
    # cancel the last search match
    def update_iter(self,widget=None):
        self.start_iter =  self.buf.get_start_iter()
        self.end_iter = self.buf.get_end_iter()
        self.current_iter = self.buf.get_iter_at_mark(self.buf.get_insert())
        self.match_start = self.match_end = None
        start, end = self.buf.get_bounds()
        self.buf.remove_tag(self.found_text_tag, start, end)

    # This will search the buffer for a specified text string.
    # You can search forward or back, with mixed case or exact text.
    # if it searches to either end, if search is pressed again, it will start at the other end.
    # This will grab focus and set the cursor active, while highlighting the line.
    # It automatically scrolls if it must.
    # it primes self.match_start for replacing text
    def text_search(self,direction=True,mixed_case=True,text="t"):
        CASEFLAG = 0
        if mixed_case:
            CASEFLAG = gtksourceview.SEARCH_CASE_INSENSITIVE
        if direction:
            if self.current_iter.is_end():
                self.current_iter = self.start_iter.copy()
            found = gtksourceview.iter_forward_search(self.current_iter,text,CASEFLAG, None)
        else:
            if self.current_iter.is_start():
                self.current_iter = self.end_iter.copy()
            found = gtksourceview.iter_backward_search(self.current_iter,text,CASEFLAG, None)
        if found:
            self.match_start,self.match_end = found
            self.buf.apply_tag(self.found_text_tag, self.match_start, self.match_end)
            self.buf.select_range(self.match_start,self.match_end)

            if direction:
                self.buf.place_cursor(self.match_start)
                self.grab_focus()
                self.current_iter = self.match_end.copy()
            else:
                self.buf.place_cursor(self.match_start)
                self.grab_focus()
                self.current_iter = self.match_start.copy()
            self.scroll_to_iter(self.match_start, 0, True, 0, 0.5)
            self.set_highlight_current_line(True)
        else:
            self.current_iter = self.start_iter.copy()
            self.set_highlight_current_line(False)
            self.match_start = self.match_end = None

    # check if we already have a match
    # if so and we are replacing-all, delete and insert without individular undo moves
    # if so but not replace-all, delete and insert with individulat undo moves
    # do a search to prime self.match_start
    # if we have gone to the end, stop searching
    # if not replace-all stop searching, otherwise start again
    def replace_text_search(self,direction=True,mixed_case=True,text="t",re_text="T",replace_all=False):
        while True:
            if self.match_start:
                if replace_all:
                    self.buf.delete(self.match_start, self.match_end)
                    self.buf.insert_at_cursor(re_text)
                else:
                    self.buf.delete_interactive(self.match_start, self.match_end,True)
                    self.buf.insert_interactive_at_cursor(re_text,True)
            self.text_search(direction,mixed_case,text)
            if self.current_iter.is_start(): break
            if not replace_all: break

    # undo one level of changes
    def undo(self):
        if self.buf.can_undo():
            self.buf.undo()

    # redo one level of changes
    def redo(self):
        if self.buf.can_redo():
            self.buf.redo()

def safe_write(filename, data, mode=0644):
    import os, tempfile
    fd, fn = tempfile.mkstemp(dir=os.path.dirname(filename), prefix=os.path.basename(filename))
    try:
        os.write(fd, data)
        os.close(fd)
        fd = None
        os.rename(fn, filename)
    finally:
        if fd is not None:
            os.close(fd)
        if os.path.isfile(fn):
            os.unlink(fn)

class EMC_Action_Save(_EMC_Action, _EMC_FileChooser):
    __gtype_name__ = 'EMC_Action_Save'
    __gproperties__ = { 'textview' : (EMC_SourceView.__gtype__, 'Textview',
                    "Corresponding textview widget", gobject.PARAM_READWRITE),
    }
    def __init__(self, *a, **kw):
        _EMC_Action.__init__(self, *a, **kw)
        self.textview = None

    def _hal_init(self):
        _EMC_Action._hal_init(self)

    def on_activate(self, w):
        if not self.textview or not self.textview.filename:
            return
        self.save(self.textview.filename)

    def save(self, fn):
        b = self.textview.get_buffer()
        b.set_modified(False)
        safe_write(fn, b.get_text(b.get_start_iter(), b.get_end_iter()))
        self._load_file(fn)

    def do_set_property(self, property, value):
        name = property.name.replace('-', '_')
        if name == 'textview':
            self.textview = value
        else:
            return _EMC_Action.do_set_property(self, property, value)

    def do_get_property(self, property):
        name = property.name.replace('-', '_')
        if name == 'textview':
            return self.textview
        else:
            return _EMC_Action.do_get_property(self, property)

class EMC_Action_SaveAs(EMC_Action_Save):
    __gtype_name__ = 'EMC_Action_SaveAs'

    def __init__(self, *a, **kw):
        _EMC_Action.__init__(self, *a, **kw)
        self.textview = None
        self.currentfolder = os.path.expanduser("~/linuxcnc/nc_files")

    def on_activate(self, w):
        if not self.textview:
            return
        dialog = gtk.FileChooserDialog(title="Save As",action=gtk.FILE_CHOOSER_ACTION_SAVE,
                    buttons=(gtk.STOCK_CANCEL,gtk.RESPONSE_CANCEL,gtk.STOCK_SAVE,gtk.RESPONSE_OK))
        dialog.set_do_overwrite_confirmation(True)
        dialog.set_current_folder(self.currentfolder)
        if self.textview.filename:
            dialog.set_current_name(os.path.basename(self.textview.filename))
        dialog.show()
        r = dialog.run()
        fn = dialog.get_filename()
        dialog.destroy()
        if r == gtk.RESPONSE_OK:
            self.save(fn)
            self.currentfolder = os.path.dirname(fn)
<commit_msg>Add option to disable line number reset in hal_sourceview when idle.<commit_after>#!/usr/bin/env python
# vim: sts=4 sw=4 et
# GladeVcp actions
#
# Copyright (c) 2011  Pavel Shramov <shramov@mexmat.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

import os, time

import gobject, gtk

from hal_widgets import _HalWidgetBase
import linuxcnc
from hal_glib import GStat
from hal_actions import _EMC_ActionBase, _EMC_Action
from hal_filechooser import _EMC_FileChooser

import gtksourceview2 as gtksourceview

class EMC_SourceView(gtksourceview.View, _EMC_ActionBase):
    __gtype_name__ = 'EMC_SourceView'
    __gproperties__ = {
        'idle_line_reset' : ( gobject.TYPE_BOOLEAN, 'Reset Line Number when idle', 'Sets line number back to 0 when code is not running or paused',
                    True, gobject.PARAM_READWRITE | gobject.PARAM_CONSTRUCT)
    }
    def __init__(self, *a, **kw):
        gtksourceview.View.__init__(self, *a, **kw)
        self.filename = None
        self.mark = None
        self.offset = 0
        self.program_length = 0
        self.idle_line_reset = True
        self.buf = gtksourceview.Buffer()
        self.buf.set_max_undo_levels(20)
        self.buf.connect('changed', self.update_iter)
        self.set_buffer(self.buf)
        self.lm = gtksourceview.LanguageManager()
        self.sm = gtksourceview.StyleSchemeManager()
        if 'EMC2_HOME' in os.environ:
            path = os.path.join(os.environ['EMC2_HOME'], 'share/gtksourceview-2.0/language-specs/')
            self.lm.set_search_path(self.lm.get_search_path() + [path])

        self.buf.set_language(self.lm.get_language('.ngc'))
        self.set_show_line_numbers(True)
        self.set_show_line_marks(True)
        self.set_highlight_current_line(True)
        self.set_mark_category_icon_from_icon_name('motion', 'gtk-forward')
        self.set_mark_category_background('motion', gtk.gdk.Color('#ff0'))
        self.found_text_tag = self.buf.create_tag(background="yellow")
        self.update_iter()
        self.connect('button-release-event', self.button_pressed)

    def do_get_property(self, property):
        name = property.name.replace('-', '_')
        if name in ['idle_line_reset']:
            return getattr(self, name)
        else:
            raise AttributeError('unknown property %s' % property.name)

    def do_set_property(self, property, value):
        name = property.name.replace('-', '_')
        if name in ['idle_line_reset']:
            return setattr(self, name, value)
        else:
            raise AttributeError('unknown property %s' % property.name)

    def _hal_init(self):
        _EMC_ActionBase._hal_init(self)
        self.gstat.connect('file-loaded', lambda w, f: gobject.timeout_add(1, self.load_file, f))
        self.gstat.connect('line-changed', self.highlight_line)
        if self.idle_line_reset:
            self.gstat.connect('interp_idle', lambda w: self.set_line_number(0))

    def set_language(self, lang, path = None):
        # path = the search path for the langauage file
        # if none, set to default
        # lang = the lang file to set
        if path == None:
            if 'EMC2_HOME' in os.environ:
                path = os.path.join(os.environ['EMC2_HOME'], 'share/gtksourceview-2.0/language-specs/')
        if path:
            self.lm.set_search_path(path)
        self.buf.set_language(self.lm.get_language(lang))

    def set_style_scheme(self, style, path = None):
        if path:
            self.sm.set_search_path(path)
        self.buf.set_style_scheme(self.sm.get_scheme(style))

    def get_filename(self):
        return self.filename

    # This load the file while not allowing undo buttons to unload the program.
    # It updates the iter because iters become invalid when anything changes.
    # We set the buffer-unmodified flag false after loading the file.
    # Set the hilight line to the line linuxcnc is looking at.
    # if one calls load_file without a filenname, We reload the exisiting file.
    def load_file(self, fn=None):
        self.buf.begin_not_undoable_action()
        if fn == None:
            fn = self.filename
        self.filename = fn
        if not fn:
            self.buf.set_text('')
            return
        self.buf.set_text(open(fn).read())
        self.buf.end_not_undoable_action()
        self.buf.set_modified(False)
        self.update_iter()
        self.highlight_line(self.gstat, self.gstat.stat.motion_line)
        self.offset = self.gstat.stat.motion_line
        f = file(fn, 'r')
        p = f.readlines()
        f.close()
        self.program_length = len(p)

    # This moves the highlight line to a lower numbered line.
    # useful for run-at-line selection
    def line_down(self):
        self.offset +=1
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    # This moves the highlight line to a higher numbered line.
    # useful for run-at-line selection
    def line_up(self):
        self.offset -=1
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    def get_line_number(self):
        return self.offset

    # sets the highlight line to a specified line.
    def set_line_number(self,linenum):
        self.offset = linenum
        self.check_offset()
        self.highlight_line(self.gstat, self.offset)

    def check_offset(self):
        if self.offset < 0:
            self.offset = 0
        elif  self.offset > self.program_length:
            self.offset = self.program_length

    def highlight_line(self, w, l):
        self.offset = l
        if not l:
            if self.mark:
                self.buf.delete_mark(self.mark)
                self.mark = None
            return
        line = self.buf.get_iter_at_line(l-1)
        if not self.mark:
            self.mark = self.buf.create_source_mark('motion', 'motion', line)
            self.mark.set_visible(True)
        else:
            self.buf.move_mark(self.mark, line)
        self.scroll_to_mark(self.mark, 0, True, 0, 0.5)

    def button_pressed(self,widget,event):
        self.update_iter()

    # iters are invalid (and will cause a complete crash) after any changes.
    # so we have to update them after a change or the user clicks on view with mouse
    # re-establish start and end of text
    # current_iter is the cursor position
    # cancel the last search match
    def update_iter(self,widget=None):
        self.start_iter =  self.buf.get_start_iter()
        self.end_iter = self.buf.get_end_iter()
        self.current_iter = self.buf.get_iter_at_mark(self.buf.get_insert())
        self.match_start = self.match_end = None
        start, end = self.buf.get_bounds()
        self.buf.remove_tag(self.found_text_tag, start, end)

    # This will search the buffer for a specified text string.
    # You can search forward or back, with mixed case or exact text.
    # if it searches to either end, if search is pressed again, it will start at the other end.
    # This will grab focus and set the cursor active, while highlighting the line.
    # It automatically scrolls if it must.
    # it primes self.match_start for replacing text
    def text_search(self,direction=True,mixed_case=True,text="t"):
        CASEFLAG = 0
        if mixed_case:
            CASEFLAG = gtksourceview.SEARCH_CASE_INSENSITIVE
        if direction:
            if self.current_iter.is_end():
                self.current_iter = self.start_iter.copy()
            found = gtksourceview.iter_forward_search(self.current_iter,text,CASEFLAG, None)
        else:
            if self.current_iter.is_start():
                self.current_iter = self.end_iter.copy()
            found = gtksourceview.iter_backward_search(self.current_iter,text,CASEFLAG, None)
        if found:
            self.match_start,self.match_end = found
            self.buf.apply_tag(self.found_text_tag, self.match_start, self.match_end)
            self.buf.select_range(self.match_start,self.match_end)

            if direction:
                self.buf.place_cursor(self.match_start)
                self.grab_focus()
                self.current_iter = self.match_end.copy()
            else:
                self.buf.place_cursor(self.match_start)
                self.grab_focus()
                self.current_iter = self.match_start.copy()
            self.scroll_to_iter(self.match_start, 0, True, 0, 0.5)
            self.set_highlight_current_line(True)
        else:
            self.current_iter = self.start_iter.copy()
            self.set_highlight_current_line(False)
            self.match_start = self.match_end = None

    # check if we already have a match
    # if so and we are replacing-all, delete and insert without individular undo moves
    # if so but not replace-all, delete and insert with individulat undo moves
    # do a search to prime self.match_start
    # if we have gone to the end, stop searching
    # if not replace-all stop searching, otherwise start again
    def replace_text_search(self,direction=True,mixed_case=True,text="t",re_text="T",replace_all=False):
        while True:
            if self.match_start:
                if replace_all:
                    self.buf.delete(self.match_start, self.match_end)
                    self.buf.insert_at_cursor(re_text)
                else:
                    self.buf.delete_interactive(self.match_start, self.match_end,True)
                    self.buf.insert_interactive_at_cursor(re_text,True)
            self.text_search(direction,mixed_case,text)
            if self.current_iter.is_start(): break
            if not replace_all: break

    # undo one level of changes
    def undo(self):
        if self.buf.can_undo():
            self.buf.undo()

    # redo one level of changes
    def redo(self):
        if self.buf.can_redo():
            self.buf.redo()

def safe_write(filename, data, mode=0644):
    import os, tempfile
    fd, fn = tempfile.mkstemp(dir=os.path.dirname(filename), prefix=os.path.basename(filename))
    try:
        os.write(fd, data)
        os.close(fd)
        fd = None
        os.rename(fn, filename)
    finally:
        if fd is not None:
            os.close(fd)
        if os.path.isfile(fn):
            os.unlink(fn)

class EMC_Action_Save(_EMC_Action, _EMC_FileChooser):
    __gtype_name__ = 'EMC_Action_Save'
    __gproperties__ = { 'textview' : (EMC_SourceView.__gtype__, 'Textview',
                    "Corresponding textview widget", gobject.PARAM_READWRITE),
    }
    def __init__(self, *a, **kw):
        _EMC_Action.__init__(self, *a, **kw)
        self.textview = None

    def _hal_init(self):
        _EMC_Action._hal_init(self)

    def on_activate(self, w):
        if not self.textview or not self.textview.filename:
            return
        self.save(self.textview.filename)

    def save(self, fn):
        b = self.textview.get_buffer()
        b.set_modified(False)
        safe_write(fn, b.get_text(b.get_start_iter(), b.get_end_iter()))
        self._load_file(fn)

    def do_set_property(self, property, value):
        name = property.name.replace('-', '_')
        if name == 'textview':
            self.textview = value
        else:
            return _EMC_Action.do_set_property(self, property, value)

    def do_get_property(self, property):
        name = property.name.replace('-', '_')
        if name == 'textview':
            return self.textview
        else:
            return _EMC_Action.do_get_property(self, property)

class EMC_Action_SaveAs(EMC_Action_Save):
    __gtype_name__ = 'EMC_Action_SaveAs'

    def __init__(self, *a, **kw):
        _EMC_Action.__init__(self, *a, **kw)
        self.textview = None
        self.currentfolder = os.path.expanduser("~/linuxcnc/nc_files")

    def on_activate(self, w):
        if not self.textview:
            return
        dialog = gtk.FileChooserDialog(title="Save As",action=gtk.FILE_CHOOSER_ACTION_SAVE,
                    buttons=(gtk.STOCK_CANCEL,gtk.RESPONSE_CANCEL,gtk.STOCK_SAVE,gtk.RESPONSE_OK))
        dialog.set_do_overwrite_confirmation(True)
        dialog.set_current_folder(self.currentfolder)
        if self.textview.filename:
            dialog.set_current_name(os.path.basename(self.textview.filename))
        dialog.show()
        r = dialog.run()
        fn = dialog.get_filename()
        dialog.destroy()
        if r == gtk.RESPONSE_OK:
            self.save(fn)
            self.currentfolder = os.path.dirname(fn)
<|endoftext|>


<commit_before>from . import aggregate
from . import amb
from . import all
from . import any
from . import average
from . import buffer
from . import count
from . import dematerialize
from . import finalvalue
from . import groupby
from . import groupbyuntil
from . import groupjoin
from . import interval
from . import join
from . import materialize
from . import observeon
from . import reduce
from . import scan
from . import select
from . import selectmany
from . import sequenceequal
from . import skip
from . import skipwhile
from . import subscribeon
from . import switchlatest
from . import take
from . import takewhile
from . import toarray
from . import where
from . import window
from . import windowwithcount<commit_msg>Moved methods to own files<commit_after>from . import aggregate
from . import all
from . import amb
from . import any
from . import average
from . import buffer
from . import count
from . import defer
from . import dematerialize
from . import empty
from . import finalvalue
from . import generate
from . import groupby
from . import groupbyuntil
from . import groupjoin
from . import interval
from . import join
from . import materialize
from . import never
from . import observeon
from . import range
from . import reduce
from . import repeat
from . import scan
from . import select
from . import selectmany
from . import sequenceequal
from . import skip
from . import skipwhile
from . import subscribeon
from . import switchlatest
from . import take
from . import takewhile
from . import timer
from . import toarray
from . import where
from . import window
from . import windowwithcount<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Prune binaries from the source tree"""

import argparse
from pathlib import Path

from _common import ENCODING, get_logger, add_common_params
import sys

def prune_dir(unpack_root, prune_files):
    """
    Delete files under unpack_root listed in prune_files. Returns an iterable of unremovable files.

    unpack_root is a pathlib.Path to the directory to be pruned
    prune_files is an iterable of files to be removed.
    """
    unremovable_files = set()
    for relative_file in prune_files:
        file_path = unpack_root / relative_file
        try:
            file_path.unlink()
        except FileNotFoundError:
            unremovable_files.add(Path(relative_file).as_posix())
    return unremovable_files

def _callback(args):
    if not args.directory.exists():
        get_logger().error('Specified directory does not exist: %s', args.directory)
        sys.exit(1)
    if not args.pruning_list.exists():
        get_logger().error('Could not find the pruning list: %s', args.pruning_list)
    prune_files = tuple(filter(len, args.pruning_list.read_text(encoding=ENCODING).splitlines()))
    unremovable_files = prune_dir(args.directory, prune_files)
    if unremovable_files:
        get_logger().error('%d files could not be pruned.', len(unremovable_files))
        get_logger().debug('Files could not be pruned:\n%s',
                           '\n'.join(f for f in unremovable_files))
        sys.exit(1)

def main():
    """CLI Entrypoint"""
    parser = argparse.ArgumentParser()
    parser.add_argument('directory', type=Path, help='The directory to apply binary pruning.')
    parser.add_argument('pruning_list', type=Path, help='Path to pruning.list')
    add_common_params(parser)
    parser.set_defaults(callback=_callback)

    args = parser.parse_args()
    args.callback(args)

if __name__ == '__main__':
    main()
<commit_msg>Add Windows compatibility for prune_binaries.py (#1592)<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Prune binaries from the source tree"""

import argparse
from pathlib import Path

from _common import ENCODING, get_logger, add_common_params
import sys
import os
import stat

def prune_dir(unpack_root, prune_files):
    """
    Delete files under unpack_root listed in prune_files. Returns an iterable of unremovable files.

    unpack_root is a pathlib.Path to the directory to be pruned
    prune_files is an iterable of files to be removed.
    """
    unremovable_files = set()
    for relative_file in prune_files:
        file_path = unpack_root / relative_file
        try:
            file_path.unlink()
        # read-only files can't be deleted on Windows
        # so remove the flag and try again.
        except PermissionError:
            os.chmod(file_path, stat.S_IWRITE)
            file_path.unlink()
        except FileNotFoundError:
            unremovable_files.add(Path(relative_file).as_posix())
    return unremovable_files

def _callback(args):
    if not args.directory.exists():
        get_logger().error('Specified directory does not exist: %s', args.directory)
        sys.exit(1)
    if not args.pruning_list.exists():
        get_logger().error('Could not find the pruning list: %s', args.pruning_list)
    prune_files = tuple(filter(len, args.pruning_list.read_text(encoding=ENCODING).splitlines()))
    unremovable_files = prune_dir(args.directory, prune_files)
    if unremovable_files:
        get_logger().error('%d files could not be pruned.', len(unremovable_files))
        get_logger().debug('Files could not be pruned:\n%s',
                           '\n'.join(f for f in unremovable_files))
        sys.exit(1)

def main():
    """CLI Entrypoint"""
    parser = argparse.ArgumentParser()
    parser.add_argument('directory', type=Path, help='The directory to apply binary pruning.')
    parser.add_argument('pruning_list', type=Path, help='Path to pruning.list')
    add_common_params(parser)
    parser.set_defaults(callback=_callback)

    args = parser.parse_args()
    args.callback(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""manages GTK3 broadwayd displays
.. and to minimize bash scripting ugggh

usage:
#displynum=display.add()
#display.app('gedit',displaynum) #where gedit is a gtk3 app

"""

import os
import subprocess
from collections import defaultdict
from time import sleep
import socket

def get_openport():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('',0))
    return s.getsockname()[1]

class sequence():
    dc=0
    @staticmethod
    def getdisplay(self):
        self.dc+=1 ;
        return self.dc
    @staticmethod
    def __call__(self):
        return self.getdisplay(self)

def friendly_display(port,begin=8000):
    """for wehn you want some 'web' ports"""
    ret= port-begin
    if ret < 0 or port<0:
        raise ValueError('neg values')
    return ret
def display_is_port(port):
    display=port
    return display
#functions need to be one to one mappings
p2df=lambda p: sequence.__call__(sequence)
#display_is_port#friendly_display#
class keydefaultdict(defaultdict):
    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError( key )
        else:
            ret = self[key] = self.default_factory(key)
            return ret

port2display={}
display2port={}
class displaydict(defaultdict):
    #adding issues are covvered by add()
    def removemapping(self,display):
        port2display.pop(display2port.pop(display))
    def __delitem__(self, display):
        super(displaydict, self).__delitem__(display)
        self.removemapping(display)
    def pop(self, display):
        super(displaydict, self).pop(display)
        self.removemapping(display)
#procs assoc with each display
running_displays=displaydict(list)

#lesson learned:
#def add(port,block=True) not a good idea to specify a port
def add(block=True):
    port=get_openport()
    remove_zombies()
    """runs the html5 part of the app returning the display number
    blocks until the dispaly server is up by default"""
    display=p2df(port)
    if display in running_displays:
        raise KeyError('display server already running')
    else:
        if isport_openable(port) is True:
            raise ValueError("can't get port "+str(port))

    try:
        p=subprocess.Popen(['./display.sh'
                           ,str(display),str(port)]
        #,preexec_fn=os.setsid
        )
    except:
        raise
    #block until 'app' is ready on the port
    if block==True:
        while ( isport_openable(port) is not True ):
            sleep(.1); continue
    #registrations
    running_displays[display].append(p)
    port2display[port]=display;
    display2port[display]=port
    # port->display should be 1 to 1 mapping
    if len(display2port) != len(port2display):
        raise Exception('display and port numbers are not 1-to-1')
    return display

def remove_zombies():
    delthese=[]
    for adisplay in running_displays:
        for an,aproc in enumerate(running_displays[adisplay]):
            if aproc.poll() is None: continue# running
            else: delthese.append( (adisplay,an) )
    for adisplay,an in delthese:
        #in case empty list
        try: running_displays[adisplay].pop(an)
        except: pass

def app(cmd,display,**kwargs):
    """runs a gtk3 prog on display. """
    remove_zombies()
    if (display) not in running_displays:
        raise ValueError('display does not exist')

    #cmd_args=list(cmd_args)
    #kwargs['preexec_fn']=os.setpgid
    sp=subprocess.Popen(['./runon_display.sh',cmd,str(display)]
                        ,**kwargs)
    running_displays[display].append(sp)
    return sp

def isport_openable(port):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.bind(('127.0.0.1',port)) #if can bind then not busy
        s.close()
        return False
    except: return True
    # cr=s.connect_ex(('127.0.0.1', port))
    # if cr==0: return True
    # else: return cr

import signal
def stop(display,signal=signal.SIGINT):
    """stops display and everything running on it"""
    if display not in running_displays:
        raise KeyError('no display #'+str(display)+' to kill')
    #os.killpg(p.pid, signal.SIGTERM)
    proclist= running_displays[display]
    for p in reversed(proclist):
        p.send_signal(signal);
        #p.kill()
        p.wait()
    running_displays.pop(display)
    remove_zombies()

def kill_all():
    """kills all display apps on the server forcefully"""
    for ad in running_displays.keys():
        stop(ad,signal=signal.SIGKILL)<commit_msg>display mgr more robust now<commit_after>#this could be in a repo on its own

"""manages GTK3 broadwayd displays
.. and to minimize bash scripting ugggh

usage:
>displynum, port =display.add()
>display.app('gedit',displaynum) #where gedit is a gtk3 app

you may want to set the limits after import
>import display
>display.DisplayLimit=10
"""

import os
import atexit
import subprocess
from collections import defaultdict
from time import sleep
import socket

class LimitError(Exception): val=None; pass
class DisplayLimit(LimitError):
    """a limit to the number of displays"""
    val=3;
    pass
class ApplicationLimit(LimitError):
    """a limit to the number of applications per display"""
    val=3
    pass

#should program onappstart onappclose
#todo capture stdio on procs

def get_openport():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('',0))
    return s.getsockname()[1]

class sequence():
    dc=0
    @staticmethod
    def getdisplay(self):
        self.dc+=1 ;
        return self.dc
    @staticmethod
    def __call__(self):
        return self.getdisplay(self)

def friendly_display(port,begin=8000):
    """for wehn you want some 'web' ports"""
    ret= port-begin
    if ret < 0 or port<0:
        raise ValueError('neg values')
    return ret

def display_is_port(port):
    display=port
    return display
#functions need to be one to one mappings
p2df=lambda p: sequence.__call__(sequence)
#display_is_port#friendly_display#
class keydefaultdict(defaultdict):
    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError( key )
        else:
            ret = self[key] = self.default_factory(key)
            return ret

port2display={}
display2port={}
class displaydict(defaultdict):
    #adding issues are covvered by add()
    def removemapping(self,display):
        port2display.pop(display2port.pop(display))
    def __delitem__(self, display):
        super(displaydict, self).__delitem__(display)
        self.removemapping(display)
    def pop(self, display):
        super(displaydict, self).pop(display)
        self.removemapping(display)
#procs assoc with each display
running_displays=displaydict(list)

#lesson learned:
#def add(port,block=True) not a good idea to specify a port
def add(portgetter=get_openport
        ,block=True):#don't see a reason to not block
    remove_zombies()
    if len(running_displays)==DisplayLimit.val:
        raise DisplayLimit(DisplayLimit.val)
    port=portgetter()
    """runs the html5 part of the app returning the display number
    blocks until the dispaly server is up by default"""
    display=p2df(port)
    if display in running_displays:
        raise KeyError('display server already running')
    else:
        if isport_openable(port) is True:
            raise ValueError("can't get port "+str(port))

    try:
        p=subprocess.Popen(['./display.sh'
                           ,str(display),str(port)]
        #,preexec_fn=os.setsid
        )
    except:
        raise Exception("couldn't start display")
    #block until 'app' is ready on the port
    if block==True:
        while ( isport_openable(port) is not True ):
            sleep(.1); continue
    #registrations
    running_displays[display].append(p) #the only reason it's a...
    #...default dict.. do i really need defaultdict?
    port2display[port]=display;
    display2port[display]=port
    # port->display should be 1 to 1 mapping
    if len(display2port) != len(port2display):
        raise Exception('display and port numbers are not 1-to-1')
    return display, port

def remove_zombies():
    #the not immediate
    delthese=[]
    for adisplay in running_displays:
        for an,aproc in enumerate(running_displays[adisplay]):
            if an==0:continue  #skip the broadway proc
            if aproc.poll() is None: continue# running
            else: delthese.append( (adisplay,an) )
    for adisplay,an in delthese:
        #in case empty list
        try: running_displays[adisplay].pop(an)
        except: pass

#what happens when the app spawns a window or another proc?
#on multiple gedits  only the first one is alive
def app(cmd,display,**kwargs):
    """runs a gtk3 prog on display. """
    if (display) not in running_displays:
        raise ValueError('display does not exist')
    remove_zombies()
    if (len(running_displays[display])-1)==ApplicationLimit.val:
        raise ApplicationLimit(ApplicationLimit.val)
    #kwargs['preexec_fn']=os.setpgid
    sp=subprocess.Popen(['./runon_display.sh',cmd,str(display)]
                        ,**kwargs)
    running_displays[display].append(sp)
    return sp

def isport_openable(port):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.bind(('127.0.0.1',port)) #if can bind then not busy
        s.close()
        return False
    except: return True
    # cr=s.connect_ex(('127.0.0.1', port))
    # if cr==0: return True
    # else: return cr

import signal
def stop(display,signal=signal.SIGINT):
    """stops display and everything running on it"""
    if display not in running_displays:
        raise KeyError('no display #'+str(display)+' to kill')
    #os.killpg(p.pid, signal.SIGTERM)
    proclist= running_displays[display]
    for p in reversed(proclist):
        p.send_signal(signal);
        #p.kill()
        p.wait()
    running_displays.pop(display)
    remove_zombies()

def kill_all():
    """kills all display apps on the server forcefully"""
    for ad in running_displays.keys():
        stop(ad,signal=signal.SIGKILL)

atexit.register(kill_all)<|endoftext|>