<commit_before>STUDENT_UPLOAD = '''
    SELECT UNIQUE
        TRIM(id_rec.firstname) firstname, id_rec.middlename[1,1] middleinitial,
        TRIM(id_rec.lastname) lastname, TRIM(id_rec.suffixname) suffix,
        id_rec.id ExternalID, 'United States' as Country,
        'Carthage College' as BusinessName, 'Students' as RecordType,
        ens_rec.phone as Phone1, 'United States' as PhoneCountry1,
        TRIM(email_rec.line1) as EmailAddress1, TRIM(ens_rec.line1) ||
        TRIM(ens_rec.line2) as EmailAddress2,
        CASE WHEN NVL(ens_rec.opt_out, 1)   =   1   THEN    ens_rec.phone
                                                    ELSE    ''
        END as SMS1, 'United States' as SMS1Country, 'Standing' as CustomField1,
        CASE prog_enr_rec.cl
            WHEN    'FF'    THEN    'Freshman'
            WHEN    'FN'    THEN    'Freshman'
            WHEN    'FR'    THEN    'Freshman'
            WHEN    'SO'    THEN    'Sophomore'
            WHEN    'JR'    THEN    'Junior'
            WHEN    'SR'    THEN    'Senior'
            WHEN    'GR'    THEN    'Graduate Student'
            WHEN    'UT'    THEN    'Transfer'
                            ELSE    'Bad match: ' || prog_enr_rec.cl
        END as CustomValue1, 'Dormitory' as CustomField2,
        CASE
            WHEN    UPPER(stu_serv_rec.bldg) = 'ABRD'       THEN    'Study Abroad'
            WHEN    UPPER(stu_serv_rec.bldg) = 'BW'         THEN    'Best Western'
            WHEN    UPPER(stu_serv_rec.bldg) = 'APT'        THEN    'Campus Apartments'
            WHEN    UPPER(stu_serv_rec.bldg) = 'CCHI'       THEN    'Chicago Programs'
            WHEN    UPPER(stu_serv_rec.bldg) = 'CMTR'       THEN    'Commuter'
            WHEN    UPPER(stu_serv_rec.bldg) = 'DEN'        THEN    'Denhart'
            WHEN    UPPER(stu_serv_rec.bldg) = 'JOH'        THEN    'Johnson'
            WHEN    UPPER(stu_serv_rec.bldg) = 'MADR'       THEN    'Madrigrano'
            WHEN    UPPER(stu_serv_rec.bldg[1,3]) = 'OAK'   THEN    'Oaks ' || stu_serv_rec.bldg[4,4]
            WHEN    UPPER(stu_serv_rec.bldg) = 'OFF'        THEN    'Commuter'
            WHEN    UPPER(stu_serv_rec.bldg) = 'SWE'        THEN    'Swenson'
            WHEN    UPPER(stu_serv_rec.bldg) = 'TAR'        THEN    'Tarble'
            WHEN    UPPER(stu_serv_rec.bldg) = 'TOWR'       THEN    'Tower'
            WHEN    UPPER(stu_serv_rec.bldg) = 'UN'         THEN    ''
            WHEN    UPPER(stu_serv_rec.bldg) = 'UNDE'       THEN    ''
            WHEN    UPPER(stu_serv_rec.bldg) = ''           THEN    ''
                                                            ELSE    'Bad match: ' || stu_serv_rec.bldg
        END AS CustomValue2,
        'Parking Lot' as CustomField3, --TRIM(REPLACE(lot_table.txt, 'Apts', '')) AS CustomValue3
        CASE WHEN TRIM(NVL(PLT.lot_code,'')) = 'CMTR' THEN 'CMTR' WHEN TRIM(NVL(PLT.lot_code,'')) <> '' THEN PLT.lot_code[1,3] || ' ' || PLT.lot_code[4,4] ELSE '' END AS CustomValue3
        ,'END' as END
    FROM
        prog_enr_rec    INNER JOIN  id_rec              ON  prog_enr_rec.id =   id_rec.id
                        LEFT JOIN   profile_rec         ON  id_rec.id       =   profile_rec.id
                        LEFT JOIN   aa_rec as email_rec ON  id_rec.id       =   email_rec.id
                                                        AND email_rec.aa    =   "EML1"
                        LEFT JOIN   stu_acad_rec        ON  id_rec.id       =   stu_acad_rec.id
                        LEFT JOIN   aa_rec as ens_rec   ON  id_rec.id       =   ens_rec.id
                                                        AND    ens_rec.aa      =   'ENS'
                        LEFT JOIN
                        (
                            SELECT stu_serv_rec.id, MAX(stu_serv_rec.stusv_no) stusv_no
                            FROM stu_serv_rec
                            GROUP BY stu_serv_rec.id
                        )           building            ON  id_rec.id           =   building.id
                        LEFT JOIN   stu_serv_rec        ON  building.stusv_no   =   stu_serv_rec.stusv_no
                        LEFT JOIN    cc_prkg_vehicle_rec    PRK    ON    id_rec.id    =    PRK.carthage_id
                                                            AND    TODAY    BETWEEN    PRK.reg_date AND NVL(PRK.inactive_date,TODAY)
                        LEFT JOIN    cc_prkg_assign_rec    PAR    ON    PRK.veh_no    =    PAR.veh_no
                                                            AND    TODAY    BETWEEN    PAR.assign_begin AND NVL(PAR.assign_end, TODAY)
                        LEFT JOIN    cc_prkg_lot_table    PLT    ON    PAR.lot_no    =    PLT.lot_no
                        --LEFT JOIN   prkgpermt_rec   prk ON  id_rec.id           =   prk.permt_id
                        --                                AND prk.acadyr          =   YEAR(TODAY)
                        --LEFT JOIN   lot_table           ON  prk.lotcode         =   lot_table.lotcode
    WHERE prog_enr_rec.subprog NOT IN
        ("YOP","UWPK","RSBD","SLS","PARA","MSW","KUSD","ENRM","CONF","CHWK")
    AND prog_enr_rec.lv_date IS NULL
    AND prog_enr_rec.cl IN ("FF","FN","FR","SO","JR","SR","GR","UT")
    AND prog_enr_rec.acst = "GOOD"
    AND stu_acad_rec.yr = YEAR(TODAY)
    AND stu_acad_rec.sess IN ("RA","RC","AM","GC","PC","TC")
    AND stu_acad_rec.reg_hrs > 0
    ORDER BY lastname, firstname
'''

ADULT_UPLOAD = '''
    SELECT
        TRIM(IDrec.firstname) AS firstname, IDrec.middlename[1,1] AS middleinitial,
        TRIM(IDrec.lastname) AS lastname, TRIM(IDrec.suffixname) AS suffix,
        IDrec.id ExternalID, 'United States' AS Country,
        'Carthage College' AS BusinessName, 'Adult Students' AS RecordType,
        ENSrec.phone AS Phone1, 'United States' AS PhoneCountry1,
        TRIM(EMLrec.line1) AS EmailAddress1,
        TRIM(ENSrec.line1) || TRIM(ENSrec.line2) AS EmailAddress2,
        CASE
            WHEN NVL(ENSrec.opt_out, 1) =    1 THEN ENSrec.phone
                                                ELSE    ''
        END AS SMS1, 'United States' AS SMS1Country, 'Standing' AS CustomField1,
        'Adult Student' AS CustomValue1, 'Dormitory' as CustomField2,
        'Commuter' AS CustomValue2, 'Parking Lot' AS CustomField3,
        'CMTR' AS CustomValue3,'END' AS END
    FROM
        id_rec  IDrec   INNER JOIN  stu_acad_rec SArec   ON  IDrec.id   =   SArec.id
                        INNER JOIN  acad_cal_rec    ACrec   ON  SArec.yr    =   ACrec.yr
                                                            AND SArec.sess  =   ACrec.sess
                        LEFT JOIN   aa_rec          ENSrec  ON  IDrec.id    =   ENSrec.id
                                                            AND ENSrec.aa   =   'ENS'
                                                            AND TODAY   BETWEEN ENSrec.beg_date AND NVL(ENSrec.end_date, TODAY)
                        LEFT JOIN   aa_rec          EMLrec  ON  IDrec.id    =   EMLrec.id
                                                            AND EMLrec.aa   =   'EML1'
                                                            AND TODAY   BETWEEN EMLrec.beg_date AND NVL(EMLrec.end_date, TODAY)
    WHERE
        ACrec.yr    =   YEAR(TODAY)
    AND
        ACrec.sess  IN  (SELECT sess FROM sess_table WHERE prog IN ('GRAD','PARA','PRDV')
        AND TODAY BETWEEN active_date AND NVL(inactive_date, TODAY))
    AND
        TODAY   BETWEEN ACrec.beg_date AND ACrec.end_date
    GROUP BY
        IDrec.id, IDrec.firstname, IDrec.lastname, IDrec.middlename[1,1],
        IDrec.suffixname, Phone1, EmailAddress1, EmailAddress2, SMS1
    ORDER BY
        lastname, firstname
'''

FACSTAFF_UPLOAD = '''
    SELECT UNIQUE
        TRIM(id_rec.firstname) firstname, id_rec.middlename[1,1] middleinitial,
        TRIM(id_rec.lastname) lastname, TRIM(id_rec.suffixname) suffix,
        id_rec.id::varchar(10) ExternalID, 'United States' as Country,
        'Carthage College' as BusinessName, 'Employee' as RecordType,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    ens_rec.phone
                                                    ELSE    ''
        END as Phone1, 'United States' as PhoneCountry1,
        CASE WHEN   school_rec.phone    =   '___-___-____'  THEN    ''
                                                            ELSE    TRIM(school_rec.phone)
        END as Phone2, 'United States' as PhoneCountry2,
        TRIM(email_rec.line1) as EmailAddress1,
        TRIM(ens_rec.line1) || TRIM(ens_rec.line2) as EmailAddress2,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    TRIM(ens_rec.phone)
                                                    ELSE    ''
        END as SMS1, 'United States' as SMS1Country, 'Office Building' as CustomField1,
        CASE TRIM(UPPER(school_rec.line3[1,3]))
            WHEN    'ARE'   THEN    'Tarble Center'
            WHEN    'CC'    THEN    'Clausen Center'
            WHEN    'DIN'   THEN    ''
            WHEN    'DSC'   THEN    'David Straz Center'
            WHEN    'HL'    THEN    'Hedberg Library'
            WHEN    'JAC'   THEN    'Johnson Art Center'
            WHEN    'JOH'   THEN    'Johnson Hall'
            WHEN    'LH'    THEN    'Lentz Hall'
            WHEN    'LEN'   THEN    'Lentz Hall'
            WHEN    'MAD'   THEN    'Madrigrano Hall'
            WHEN    'PEC'   THEN    'Physical Education Center'
            WHEN    'SC'    THEN    'Siebert Chapel'
            WHEN    'SIE'   THEN    'Siebert Chapel'
            WHEN    'STR'   THEN    'David Straz Center'
            WHEN    'SU'    THEN    'Student Union'
            WHEN    'TAR'   THEN    'Tarble Center'
            WHEN    'TWC'   THEN    'Todd Wehr Center'
            WHEN    ''      THEN    ''
                            ELSE    'Bad match: ' || TRIM(REPLACE(school_rec.line3,',','[comma]'))
        END as CustomValue1, 'Standing' as CustomField2,
        CASE jenzcst_rec.status_code
            WHEN    'FAC'   THEN    'Faculty'
            WHEN    'STF'   THEN    'Staff'
        END as CustomValue2, 'Full/Part Time' as CustomField3,
        CASE hrstat.hrstat
            WHEN    'FT'    THEN    'Full-Time'
            WHEN    'PT'    THEN    'Part-Time'
        END AS CustomValue3
        ,'END' as END
    FROM
        id_rec  INNER JOIN  job_rec             ON  id_rec.id       =   job_rec.id
                INNER JOIN  pos_table           ON  job_rec.tpos_no =   pos_table.tpos_no
                LEFT JOIN   aa_rec  school_rec  ON  id_rec.id       =   school_rec.id
                                                AND school_rec.aa   =   "SCHL"
                LEFT JOIN   aa_rec  email_rec   ON  id_rec.id       =   email_rec.id
                                                AND email_rec.aa    =   "EML1"
                LEFT JOIN   aa_rec  ens_rec     ON  id_rec.id       =   ens_rec.id
                                                AND ens_rec.aa      =   'ENS'
                LEFT JOIN
                (
                    SELECT  host_id, MIN(seq_no) seq_no
                    FROM    jenzcst_rec
                    WHERE   status_code IN ('FAC','STF')
                    GROUP BY    host_id
                )   filter                      ON  id_rec.id       =   filter.host_id
                LEFT JOIN   jenzcst_rec         ON  filter.host_id  =   jenzcst_rec.host_id
                                                AND filter.seq_no   =   jenzcst_rec.seq_no
                LEFT JOIN
                (
                    SELECT  id, hrstat
                    FROM    job_rec
                    WHERE   TODAY   BETWEEN beg_date    AND NVL(end_date, TODAY)
                    AND     hrstat  IN      ('FT','PT')
                    GROUP BY id, hrstat
                )   hrstat                      ON  id_rec.id       =   hrstat.id
    WHERE
        TODAY   BETWEEN job_rec.beg_date        AND NVL(job_rec.end_date, TODAY)
        AND
        TODAY   BETWEEN pos_table.active_date   AND NVL(pos_table.inactive_date, TODAY)
        AND
        job_rec.title_rank  IS  NOT NULL
    ORDER BY lastname, firstname
'''
<commit_msg>manually adding a contact to the data<commit_after>STUDENT_UPLOAD = '''
    SELECT UNIQUE
        TRIM(id_rec.firstname) firstname, id_rec.middlename[1,1] middleinitial,
        TRIM(id_rec.lastname) lastname, TRIM(id_rec.suffixname) suffix,
        id_rec.id ExternalID, 'United States' as Country,
        'Carthage College' as BusinessName, 'Students' as RecordType,
        ens_rec.phone as Phone1, 'United States' as PhoneCountry1,
        TRIM(email_rec.line1) as EmailAddress1, TRIM(ens_rec.line1) ||
        TRIM(ens_rec.line2) as EmailAddress2,
        CASE WHEN NVL(ens_rec.opt_out, 1)   =   1   THEN    ens_rec.phone
                                                    ELSE    ''
        END as SMS1, 'United States' as SMS1Country, 'Standing' as CustomField1,
        CASE prog_enr_rec.cl
            WHEN    'FF'    THEN    'Freshman'
            WHEN    'FN'    THEN    'Freshman'
            WHEN    'FR'    THEN    'Freshman'
            WHEN    'SO'    THEN    'Sophomore'
            WHEN    'JR'    THEN    'Junior'
            WHEN    'SR'    THEN    'Senior'
            WHEN    'GR'    THEN    'Graduate Student'
            WHEN    'UT'    THEN    'Transfer'
                            ELSE    'Bad match: ' || prog_enr_rec.cl
        END as CustomValue1, 'Dormitory' as CustomField2,
        CASE
            WHEN    UPPER(stu_serv_rec.bldg) = 'ABRD'       THEN    'Study Abroad'
            WHEN    UPPER(stu_serv_rec.bldg) = 'BW'         THEN    'Best Western'
            WHEN    UPPER(stu_serv_rec.bldg) = 'APT'        THEN    'Campus Apartments'
            WHEN    UPPER(stu_serv_rec.bldg) = 'CCHI'       THEN    'Chicago Programs'
            WHEN    UPPER(stu_serv_rec.bldg) = 'CMTR'       THEN    'Commuter'
            WHEN    UPPER(stu_serv_rec.bldg) = 'DEN'        THEN    'Denhart'
            WHEN    UPPER(stu_serv_rec.bldg) = 'JOH'        THEN    'Johnson'
            WHEN    UPPER(stu_serv_rec.bldg) = 'MADR'       THEN    'Madrigrano'
            WHEN    UPPER(stu_serv_rec.bldg[1,3]) = 'OAK'   THEN    'Oaks ' || stu_serv_rec.bldg[4,4]
            WHEN    UPPER(stu_serv_rec.bldg) = 'OFF'        THEN    'Commuter'
            WHEN    UPPER(stu_serv_rec.bldg) = 'SWE'        THEN    'Swenson'
            WHEN    UPPER(stu_serv_rec.bldg) = 'TAR'        THEN    'Tarble'
            WHEN    UPPER(stu_serv_rec.bldg) = 'TOWR'       THEN    'Tower'
            WHEN    UPPER(stu_serv_rec.bldg) = 'UN'         THEN    ''
            WHEN    UPPER(stu_serv_rec.bldg) = 'UNDE'       THEN    ''
            WHEN    UPPER(stu_serv_rec.bldg) = ''           THEN    ''
                                                            ELSE    'Bad match: ' || stu_serv_rec.bldg
        END AS CustomValue2,
        'Parking Lot' as CustomField3, --TRIM(REPLACE(lot_table.txt, 'Apts', '')) AS CustomValue3
        CASE WHEN TRIM(NVL(PLT.lot_code,'')) = 'CMTR' THEN 'CMTR' WHEN TRIM(NVL(PLT.lot_code,'')) <> '' THEN PLT.lot_code[1,3] || ' ' || PLT.lot_code[4,4] ELSE '' END AS CustomValue3
        ,'END' as END
    FROM
        prog_enr_rec    INNER JOIN  id_rec              ON  prog_enr_rec.id =   id_rec.id
                        LEFT JOIN   profile_rec         ON  id_rec.id       =   profile_rec.id
                        LEFT JOIN   aa_rec as email_rec ON  id_rec.id       =   email_rec.id
                                                        AND email_rec.aa    =   "EML1"
                        LEFT JOIN   stu_acad_rec        ON  id_rec.id       =   stu_acad_rec.id
                        LEFT JOIN   aa_rec as ens_rec   ON  id_rec.id       =   ens_rec.id
                                                        AND    ens_rec.aa      =   'ENS'
                        LEFT JOIN
                        (
                            SELECT stu_serv_rec.id, MAX(stu_serv_rec.stusv_no) stusv_no
                            FROM stu_serv_rec
                            GROUP BY stu_serv_rec.id
                        )           building            ON  id_rec.id           =   building.id
                        LEFT JOIN   stu_serv_rec        ON  building.stusv_no   =   stu_serv_rec.stusv_no
                        LEFT JOIN    cc_prkg_vehicle_rec    PRK    ON    id_rec.id    =    PRK.carthage_id
                                                            AND    TODAY    BETWEEN    PRK.reg_date AND NVL(PRK.inactive_date,TODAY)
                        LEFT JOIN    cc_prkg_assign_rec    PAR    ON    PRK.veh_no    =    PAR.veh_no
                                                            AND    TODAY    BETWEEN    PAR.assign_begin AND NVL(PAR.assign_end, TODAY)
                        LEFT JOIN    cc_prkg_lot_table    PLT    ON    PAR.lot_no    =    PLT.lot_no
                        --LEFT JOIN   prkgpermt_rec   prk ON  id_rec.id           =   prk.permt_id
                        --                                AND prk.acadyr          =   YEAR(TODAY)
                        --LEFT JOIN   lot_table           ON  prk.lotcode         =   lot_table.lotcode
    WHERE prog_enr_rec.subprog NOT IN
        ("YOP","UWPK","RSBD","SLS","PARA","MSW","KUSD","ENRM","CONF","CHWK")
    AND prog_enr_rec.lv_date IS NULL
    AND prog_enr_rec.cl IN ("FF","FN","FR","SO","JR","SR","GR","UT")
    AND prog_enr_rec.acst = "GOOD"
    AND stu_acad_rec.yr = YEAR(TODAY)
    AND stu_acad_rec.sess IN ("RA","RC","AM","GC","PC","TC")
    AND stu_acad_rec.reg_hrs > 0
    ORDER BY lastname, firstname
'''

ADULT_UPLOAD = '''
    SELECT
        TRIM(IDrec.firstname) AS firstname, IDrec.middlename[1,1] AS middleinitial,
        TRIM(IDrec.lastname) AS lastname, TRIM(IDrec.suffixname) AS suffix,
        IDrec.id ExternalID, 'United States' AS Country,
        'Carthage College' AS BusinessName, 'Adult Students' AS RecordType,
        ENSrec.phone AS Phone1, 'United States' AS PhoneCountry1,
        TRIM(EMLrec.line1) AS EmailAddress1,
        TRIM(ENSrec.line1) || TRIM(ENSrec.line2) AS EmailAddress2,
        CASE
            WHEN NVL(ENSrec.opt_out, 1) =    1 THEN ENSrec.phone
                                                ELSE    ''
        END AS SMS1, 'United States' AS SMS1Country, 'Standing' AS CustomField1,
        'Adult Student' AS CustomValue1, 'Dormitory' as CustomField2,
        'Commuter' AS CustomValue2, 'Parking Lot' AS CustomField3,
        'CMTR' AS CustomValue3,'END' AS END
    FROM
        id_rec  IDrec   INNER JOIN  stu_acad_rec SArec   ON  IDrec.id   =   SArec.id
                        INNER JOIN  acad_cal_rec    ACrec   ON  SArec.yr    =   ACrec.yr
                                                            AND SArec.sess  =   ACrec.sess
                        LEFT JOIN   aa_rec          ENSrec  ON  IDrec.id    =   ENSrec.id
                                                            AND ENSrec.aa   =   'ENS'
                                                            AND TODAY   BETWEEN ENSrec.beg_date AND NVL(ENSrec.end_date, TODAY)
                        LEFT JOIN   aa_rec          EMLrec  ON  IDrec.id    =   EMLrec.id
                                                            AND EMLrec.aa   =   'EML1'
                                                            AND TODAY   BETWEEN EMLrec.beg_date AND NVL(EMLrec.end_date, TODAY)
    WHERE
        ACrec.yr    =   YEAR(TODAY)
    AND
        ACrec.sess  IN  (SELECT sess FROM sess_table WHERE prog IN ('GRAD','PARA','PRDV')
        AND TODAY BETWEEN active_date AND NVL(inactive_date, TODAY))
    AND
        TODAY   BETWEEN ACrec.beg_date AND ACrec.end_date
    GROUP BY
        IDrec.id, IDrec.firstname, IDrec.lastname, IDrec.middlename[1,1],
        IDrec.suffixname, Phone1, EmailAddress1, EmailAddress2, SMS1
    ORDER BY
        lastname, firstname
'''

FACSTAFF_UPLOAD = '''
    SELECT UNIQUE
        TRIM(id_rec.firstname) firstname, id_rec.middlename[1,1] middleinitial,
        TRIM(id_rec.lastname) lastname, TRIM(id_rec.suffixname) suffix,
        id_rec.id::varchar(10) ExternalID, 'United States' as Country,
        'Carthage College' as BusinessName, 'Employee' as RecordType,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    ens_rec.phone
                                                    ELSE    ''
        END as Phone1, 'United States' as PhoneCountry1,
        CASE WHEN   school_rec.phone    =   '___-___-____'  THEN    ''
                                                            ELSE    TRIM(school_rec.phone)
        END as Phone2, 'United States' as PhoneCountry2,
        TRIM(email_rec.line1) as EmailAddress1,
        TRIM(ens_rec.line1) || TRIM(ens_rec.line2) as EmailAddress2,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    TRIM(ens_rec.phone)
                                                    ELSE    ''
        END as SMS1, 'United States' as SMS1Country, 'Office Building' as CustomField1,
        CASE TRIM(UPPER(school_rec.line3[1,3]))
            WHEN    'ARE'   THEN    'Tarble Center'
            WHEN    'CC'    THEN    'Clausen Center'
            WHEN    'DIN'   THEN    ''
            WHEN    'DSC'   THEN    'David Straz Center'
            WHEN    'HL'    THEN    'Hedberg Library'
            WHEN    'JAC'   THEN    'Johnson Art Center'
            WHEN    'JOH'   THEN    'Johnson Hall'
            WHEN    'LH'    THEN    'Lentz Hall'
            WHEN    'LEN'   THEN    'Lentz Hall'
            WHEN    'MAD'   THEN    'Madrigrano Hall'
            WHEN    'PEC'   THEN    'Physical Education Center'
            WHEN    'SC'    THEN    'Siebert Chapel'
            WHEN    'SIE'   THEN    'Siebert Chapel'
            WHEN    'STR'   THEN    'David Straz Center'
            WHEN    'SU'    THEN    'Student Union'
            WHEN    'TAR'   THEN    'Tarble Center'
            WHEN    'TWC'   THEN    'Todd Wehr Center'
            WHEN    ''      THEN    ''
                            ELSE    'Bad match: ' || TRIM(REPLACE(school_rec.line3,',','[comma]'))
        END as CustomValue1, 'Standing' as CustomField2,
        CASE jenzcst_rec.status_code
            WHEN    'FAC'   THEN    'Faculty'
            WHEN    'STF'   THEN    'Staff'
        END as CustomValue2, 'Full/Part Time' as CustomField3,
        CASE hrstat.hrstat
            WHEN    'FT'    THEN    'Full-Time'
            WHEN    'PT'    THEN    'Part-Time'
        END AS CustomValue3
        ,'END' as END
    FROM
        id_rec  INNER JOIN  job_rec             ON  id_rec.id       =   job_rec.id
                INNER JOIN  pos_table           ON  job_rec.tpos_no =   pos_table.tpos_no
                LEFT JOIN   aa_rec  school_rec  ON  id_rec.id       =   school_rec.id
                                                AND school_rec.aa   =   "SCHL"
                LEFT JOIN   aa_rec  email_rec   ON  id_rec.id       =   email_rec.id
                                                AND email_rec.aa    =   "EML1"
                LEFT JOIN   aa_rec  ens_rec     ON  id_rec.id       =   ens_rec.id
                                                AND ens_rec.aa      =   'ENS'
                LEFT JOIN
                (
                    SELECT  host_id, MIN(seq_no) seq_no
                    FROM    jenzcst_rec
                    WHERE   status_code IN ('FAC','STF')
                    GROUP BY    host_id
                )   filter                      ON  id_rec.id       =   filter.host_id
                LEFT JOIN   jenzcst_rec         ON  filter.host_id  =   jenzcst_rec.host_id
                                                AND filter.seq_no   =   jenzcst_rec.seq_no
                LEFT JOIN
                (
                    SELECT  id, hrstat
                    FROM    job_rec
                    WHERE   TODAY   BETWEEN beg_date    AND NVL(end_date, TODAY)
                    AND     hrstat  IN      ('FT','PT')
                    GROUP BY id, hrstat
                )   hrstat                      ON  id_rec.id       =   hrstat.id
    WHERE
        TODAY   BETWEEN job_rec.beg_date        AND NVL(job_rec.end_date, TODAY)
        AND
        TODAY   BETWEEN pos_table.active_date   AND NVL(pos_table.inactive_date, TODAY)
        AND
        job_rec.title_rank  IS  NOT NULL
    UNION
    SELECT
        TRIM(id_rec.firstname) firstname, id_rec.middlename[1,1] middleinitial,
        TRIM(id_rec.lastname) lastname, TRIM(id_rec.suffixname) suffix,
        id_rec.id::varchar(10) AS ExternalID, 'United States' as Country,
        'Carthage College' as BusinessName, 'Employee' as RecordType,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    ens_rec.phone
                                                       ELSE    ''
        END as Phone1, 'United States' as PhoneCountry1,
        CASE WHEN   school_rec.phone    =   '___-___-____'  THEN    ''
                                                           ELSE    TRIM(school_rec.phone)
        END as Phone2, 'United States' as PhoneCountry2,
        TRIM(email_rec.line1) as EmailAddress1,
        TRIM(ens_rec.line1) || TRIM(ens_rec.line2) as EmailAddress2,
        CASE WHEN   NVL(ens_rec.opt_out, 1) =   1   THEN    TRIM(ens_rec.phone)
                                                   ELSE    ''
        END as SMS1, 'United States' as SMS1Country, 'Office Building' as CustomField1,
        '' AS CustomValue1, 'Standing' as CustomField2,
        'STF' AS CustomValue2, 'Full/Part Time' as CustomField3,
        'Full-Time' AS CustomValue3, 'END' as END
    FROM
        id_rec    LEFT JOIN    aa_rec    ens_rec    ON    id_rec.id        =    ens_rec.id
                                        AND    ens_rec.aa    =    'ENS'
                LEFT JOIN    aa_rec    school_rec    ON    id_rec.id    =    school_rec.id
                                            AND    school_rec.aa    =    'SCHL'
                LEFT JOIN    aa_rec    email_rec    ON    id_rec.id        =    email_rec.id
                                        AND    email_rec.aa    =    'EML1'
    WHERE
        id_rec.id    =    1518708
    ORDER BY lastname, firstname
'''
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import tensorflow as tf
import re
import os

class AbstractModule(object):
    """Superclass for DLTK core modules - strongly inspired by Sonnet: https://github.com/deepmind/sonnet

    This class wraps implements a wrapping of `tf.make_template` for automatic variable sharing. Each subclass needs to
    implement a `_build` function used for the template and call this superclass' `__init__` to create the template.
    For the variable sharing to work, variables inside `_build` have to be created via `tf.get_variable` instead of
    `tf.Variable`.

    The created template is automatically called using `__call__`.
    """
    MODEL_COLLECTIONS = [tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.MODEL_VARIABLES]
    TRAINABLE_COLLECTIONS = MODEL_COLLECTIONS + [tf.GraphKeys.TRAINABLE_VARIABLES]
    WEIGHT_COLLECTIONS = TRAINABLE_COLLECTIONS + [tf.GraphKeys.WEIGHTS]
    BIAS_COLLECTIONS = TRAINABLE_COLLECTIONS + [tf.GraphKeys.BIASES]
    MOVING_COLLECTIONS = MODEL_COLLECTIONS + [tf.GraphKeys.MOVING_AVERAGE_VARIABLES]

    def __init__(self, name=None):
        """Initialisation of the template and naming of the module

        Parameters
        ----------
        name : string
            name of the module
        """
        self.name = name
        self.variables = []
        self._template = tf.make_template(name, self._build, create_scope_now_=True)

        # Update __call__ and the object docstrings to enable better introspection (from Sonnet)
        self.__doc__ = self._build.__doc__
        self.__call__.__func__.__doc__ = self._build.__doc__

    def _build(self, *args, **kwargs):
        """Abstract function that is use to make the template when building the module

        Raises
        -------
        NotImplementedError
            This is an abstract function

        """
        raise NotImplementedError('Not implemented in abstract class')

    def __call__(self, *args, **kwargs):
        """Wrapper to call template when module is called

        Returns
        -------
        object
            Returns output of _build function

        """
        out = self._template(*args, **kwargs)

        return out

    @property
    def variable_scope(self):
        """Getter to access variable scope of the built template"""
        return self._template.variable_scope

    def get_variables(self, collection=tf.GraphKeys.TRAINABLE_VARIABLES):
        """Helper to get all variables of a given collection created within this module

        Parameters
        ----------
        collection : string, optional
            Identifier of the collection to get variables from. Defaults to `tf.GraphKeys.TRAINABLE_VARIABLES`

        Returns
        -------
        list
            List of `tf.Variables` that are part of the collection and within the scope of this module
        """
        scope_name = re.escape(self.variable_scope.name) + "/"
        return tuple(tf.get_collection(collection, scope_name))

class SaveableModule(AbstractModule):
    output_keys = []

    def __init__(self, name=None):
        self.input_placeholders = None
        self.saver = None
        super(SaveableModule, self).__init__(name)

    def _build_input_placeholder(self):
        raise NotImplementedError('Not implemented in abstract class')

    def save_metagraph(self, path, clear_devices=False, **kwargs):
        """

        Parameters
        ----------
        path : string
            path to save the metagraph to
        clear_devices : bool
            flag to toggle whether meta graph saves device placement of tensors
        kwargs
            additional arguments to the module build function

        """
        g = tf.get_default_graph()

        assert(not g.finalized, 'Graph cannot be finalized')
        assert(self.input_placeholders is not None, 'Input placeholders need to be built')

        self.saved_inputs = self.input_placeholders

        out = self._template(*self.saved_inputs, **kwargs)

        self.saved_outputs = out.values() if isinstance(out, dict) else [out]

        self.saved_var_list = list(self.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))

        self.saver = tf.train.Saver(var_list=self.saved_var_list)

        g.clear_collection('saved_network')
        g.clear_collection('saved_inputs')
        g.clear_collection('saved_outputs')

        for i in self.saved_inputs:
            g.add_to_collections(['saved_inputs', 'saved_network'], i)

        for o in self.saved_outputs:
            g.add_to_collections(['saved_outputs', 'saved_network'], o)

        for tensor in self.saved_var_list:
            g.add_to_collection('saved_network', tensor)

        self.saver.export_meta_graph('{}.meta'.format(path), clear_devices=clear_devices)

        g.clear_collection('saved_network')
        g.clear_collection('saved_inputs')
        g.clear_collection('saved_outputs')

    def save_model(self, path, session):
        """Saves the network to a given path

        Parameters
        ----------
        path : string
            Path to the file to save the network in
        session : tf.Session
            Tensorflow Sessions holding the current variable states
        """

        assert(self.saver is not None, 'Meta graph must be saved first')

        self.saver.save(session, path, write_meta_graph=False)

    @classmethod
    def load(cls, path, session):
        """

        Parameters
        ----------
        path : string
            Path to load the network from
        session : tf.Session
            Tensorflow Sessions to load the variables into

        Returns
        -------
        list : list of input placeholders saved
        list : list of outputs produced by the network

        """

        saver = tf.train.import_meta_graph('{}.meta'.format(path))

        saver.restore(session, path)

        inputs = tf.get_collection('saved_inputs')
        loaded_outputs = tf.get_collection('saved_outputs')

        outputs = {key: output for key, output in zip(cls.output_keys, loaded_outputs)}

        return inputs, outputs
<commit_msg>fix asserts<commit_after>from __future__ import division
from __future__ import absolute_import
from __future__ import print_function

import tensorflow as tf
import re
import os

class AbstractModule(object):
    """Superclass for DLTK core modules - strongly inspired by Sonnet: https://github.com/deepmind/sonnet

    This class wraps implements a wrapping of `tf.make_template` for automatic variable sharing. Each subclass needs to
    implement a `_build` function used for the template and call this superclass' `__init__` to create the template.
    For the variable sharing to work, variables inside `_build` have to be created via `tf.get_variable` instead of
    `tf.Variable`.

    The created template is automatically called using `__call__`.
    """
    MODEL_COLLECTIONS = [tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.MODEL_VARIABLES]
    TRAINABLE_COLLECTIONS = MODEL_COLLECTIONS + [tf.GraphKeys.TRAINABLE_VARIABLES]
    WEIGHT_COLLECTIONS = TRAINABLE_COLLECTIONS + [tf.GraphKeys.WEIGHTS]
    BIAS_COLLECTIONS = TRAINABLE_COLLECTIONS + [tf.GraphKeys.BIASES]
    MOVING_COLLECTIONS = MODEL_COLLECTIONS + [tf.GraphKeys.MOVING_AVERAGE_VARIABLES]

    def __init__(self, name=None):
        """Initialisation of the template and naming of the module

        Parameters
        ----------
        name : string
            name of the module
        """
        self.name = name
        self.variables = []
        self._template = tf.make_template(name, self._build, create_scope_now_=True)

        # Update __call__ and the object docstrings to enable better introspection (from Sonnet)
        self.__doc__ = self._build.__doc__
        self.__call__.__func__.__doc__ = self._build.__doc__

    def _build(self, *args, **kwargs):
        """Abstract function that is use to make the template when building the module

        Raises
        -------
        NotImplementedError
            This is an abstract function

        """
        raise NotImplementedError('Not implemented in abstract class')

    def __call__(self, *args, **kwargs):
        """Wrapper to call template when module is called

        Returns
        -------
        object
            Returns output of _build function

        """
        out = self._template(*args, **kwargs)

        return out

    @property
    def variable_scope(self):
        """Getter to access variable scope of the built template"""
        return self._template.variable_scope

    def get_variables(self, collection=tf.GraphKeys.TRAINABLE_VARIABLES):
        """Helper to get all variables of a given collection created within this module

        Parameters
        ----------
        collection : string, optional
            Identifier of the collection to get variables from. Defaults to `tf.GraphKeys.TRAINABLE_VARIABLES`

        Returns
        -------
        list
            List of `tf.Variables` that are part of the collection and within the scope of this module
        """
        scope_name = re.escape(self.variable_scope.name) + "/"
        return tuple(tf.get_collection(collection, scope_name))

class SaveableModule(AbstractModule):
    output_keys = []

    def __init__(self, name=None):
        self.input_placeholders = None
        self.saver = None
        super(SaveableModule, self).__init__(name)

    def _build_input_placeholder(self):
        raise NotImplementedError('Not implemented in abstract class')

    def save_metagraph(self, path, clear_devices=False, **kwargs):
        """

        Parameters
        ----------
        path : string
            path to save the metagraph to
        clear_devices : bool
            flag to toggle whether meta graph saves device placement of tensors
        kwargs
            additional arguments to the module build function

        """
        g = tf.get_default_graph()

        assert not g.finalized, 'Graph cannot be finalized'
        assert self.input_placeholders is not None, 'Input placeholders need to be built'

        self.saved_inputs = self.input_placeholders

        out = self._template(*self.saved_inputs, **kwargs)

        self.saved_outputs = out.values() if isinstance(out, dict) else [out]

        self.saved_var_list = list(self.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))

        self.saver = tf.train.Saver(var_list=self.saved_var_list)

        g.clear_collection('saved_network')
        g.clear_collection('saved_inputs')
        g.clear_collection('saved_outputs')

        for i in self.saved_inputs:
            g.add_to_collections(['saved_inputs', 'saved_network'], i)

        for o in self.saved_outputs:
            g.add_to_collections(['saved_outputs', 'saved_network'], o)

        for tensor in self.saved_var_list:
            g.add_to_collection('saved_network', tensor)

        self.saver.export_meta_graph('{}.meta'.format(path), clear_devices=clear_devices)

        g.clear_collection('saved_network')
        g.clear_collection('saved_inputs')
        g.clear_collection('saved_outputs')

    def save_model(self, path, session):
        """Saves the network to a given path

        Parameters
        ----------
        path : string
            Path to the file to save the network in
        session : tf.Session
            Tensorflow Sessions holding the current variable states
        """

        assert self.saver is not None, 'Meta graph must be saved first'

        self.saver.save(session, path, write_meta_graph=False)

    @classmethod
    def load(cls, path, session):
        """

        Parameters
        ----------
        path : string
            Path to load the network from
        session : tf.Session
            Tensorflow Sessions to load the variables into

        Returns
        -------
        list : list of input placeholders saved
        list : list of outputs produced by the network

        """

        saver = tf.train.import_meta_graph('{}.meta'.format(path))

        saver.restore(session, path)

        inputs = tf.get_collection('saved_inputs')
        loaded_outputs = tf.get_collection('saved_outputs')

        outputs = {key: output for key, output in zip(cls.output_keys, loaded_outputs)}

        return inputs, outputs
<|endoftext|>


<commit_before>"""Classes for describing instruction formats."""
from __future__ import absolute_import
from .operands import OperandKind, VALUE, VARIABLE_ARGS
from .operands import Operand  # noqa

# The typing module is only required by mypy, and we don't use these imports
# outside type comments.
try:
    from typing import Dict, List, Tuple, Union, Any, Sequence, Iterable  # noqa
except ImportError:
    pass

class InstructionFormat(object):
    """
    Every instruction opcode has a corresponding instruction format which
    determines the number of operands and their kinds. Instruction formats are
    identified structurally, i.e., the format of an instruction is derived from
    the kinds of operands used in its declaration.

    Most instruction formats produce a single result, or no result at all. If
    an instruction can produce more than one result, the `multiple_results`
    flag must be set on its format. All results are of the `value` kind, and
    the instruction format does not keep track of how many results are
    produced. Some instructions, like `call`, may have a variable number of
    results.

    All instruction formats must be predefined in the
    :py:mod:`cretonne.formats` module.

    :param kinds: List of `OperandKind` objects describing the operands.
    :param name: Instruction format name in CamelCase. This is used as a Rust
        variant name in both the `InstructionData` and `InstructionFormat`
        enums.
    :param multiple_results: Set to `True` if this instruction format allows
        more than one result to be produced.
    :param value_list: Set to `True` if this instruction format uses a
        `ValueList` member to store its value operands.
    :param boxed_storage: Set to `True` is this instruction format requires a
        `data: Box<...>` pointer to additional storage in its `InstructionData`
        variant.
    :param typevar_operand: Index of the input operand that is used to infer
        the controlling type variable. By default, this is the first `value`
        operand.
    """

    # Map (multiple_results, kind, kind, ...) -> InstructionFormat
    _registry = dict()  # type: Dict[Tuple[bool, Tuple[OperandKind, ...]], InstructionFormat]  # noqa

    # All existing formats.
    all_formats = list()  # type: List[InstructionFormat]

    def __init__(self, *kinds, **kwargs):
        # type: (*Union[OperandKind, Tuple[str, OperandKind]], **Any) -> None # noqa
        self.name = kwargs.get('name', None)  # type: str
        self.multiple_results = kwargs.get('multiple_results', False)
        self.has_value_list = kwargs.get('value_list', False)
        self.boxed_storage = kwargs.get('boxed_storage', False)
        self.members = list()  # type: List[str]

        # Struct member names for the immediate operands. All other instruction
        # operands are values or variable argument lists. They are all handled
        # specially.
        self.imm_members = list()  # type: List[str]
        # Operand kinds for the immediate operands.
        self.imm_kinds = list()  # type: List[OperandKind]
        # The number of value operands stored in the format, or `None` when
        # `has_value_list` is set.
        self.num_value_operands = 0

        self.kinds = tuple(self._process_member_names(kinds))

        # Which of self.kinds are `value`?
        self.value_operands = tuple(
                i for i, k in enumerate(self.kinds) if k is VALUE)

        # The typevar_operand argument must point to a 'value' operand.
        self.typevar_operand = kwargs.get('typevar_operand', None)  # type: int
        if self.typevar_operand is not None:
            assert self.kinds[self.typevar_operand] is VALUE, \
                    "typevar_operand must indicate a 'value' operand"
        elif len(self.value_operands) > 0:
            # Default to the first 'value' operand, if there is one.
            self.typevar_operand = self.value_operands[0]

        # Compute a signature for the global registry.
        sig = (self.multiple_results, self.kinds)
        if sig in InstructionFormat._registry:
            raise RuntimeError(
                "Format '{}' has the same signature as existing format '{}'"
                .format(self.name, InstructionFormat._registry[sig]))
        InstructionFormat._registry[sig] = self
        InstructionFormat.all_formats.append(self)

    def _process_member_names(self, kinds):
        # type: (Sequence[Union[OperandKind, Tuple[str, OperandKind]]]) -> Iterable[OperandKind] # noqa
        """
        Extract names of all the immediate operands in the kinds tuple.

        Each entry is either an `OperandKind` instance, or a `(member, kind)`
        pair. The member names correspond to members in the Rust
        `InstructionData` data structure.

        Yields the operand kinds.
        """
        for arg in kinds:
            if isinstance(arg, OperandKind):
                member = arg.default_member
                k = arg
            else:
                member, k = arg

            # We define 'immediate' as not a value or variable arguments.
            if k is VALUE:
                self.num_value_operands += 1
            elif k is VARIABLE_ARGS:
                # We require a value list for storage of variable arguments.
                assert self.has_value_list, "Need a value list"
            else:
                self.imm_kinds.append(k)
                self.imm_members.append(member)

            self.members.append(member)
            yield k

    def __str__(self):
        # type: () -> str
        args = ', '.join('{}: {}'.format(m, k) if m else str(k)
                         for m, k in zip(self.members, self.kinds))
        return '{}({})'.format(self.name, args)

    def __getattr__(self, attr):
        # type: (str) -> FormatField
        """
        Make instruction format members available as attributes.

        Each non-value format member becomes a corresponding `FormatField`
        attribute.
        """
        try:
            i = self.members.index(attr)
        except ValueError:
            raise AttributeError(
                    '{} is neither a {} member or a '
                    .format(attr, self.name) +
                    'normal InstructionFormat attribute')
        field = FormatField(self, i, attr)
        setattr(self, attr, field)
        return field

    @staticmethod
    def lookup(ins, outs):
        # type: (Sequence[Operand], Sequence[Operand]) -> InstructionFormat
        """
        Find an existing instruction format that matches the given lists of
        instruction inputs and outputs.

        The `ins` and `outs` arguments correspond to the
        :py:class:`Instruction` arguments of the same name, except they must be
        tuples of :py:`Operand` objects.
        """
        if len(outs) == 1:
            multiple_results = outs[0].kind == VARIABLE_ARGS
        else:
            multiple_results = len(outs) > 1
        sig = (multiple_results, tuple(op.kind for op in ins))
        if sig not in InstructionFormat._registry:
            raise RuntimeError(
                    "No instruction format matches ins = ({}){}".format(
                        ", ".join(map(str, sig[1])),
                        "[multiple results]" if multiple_results else ""))
        return InstructionFormat._registry[sig]

    @staticmethod
    def extract_names(globs):
        """
        Given a dict mapping name -> object as returned by `globals()`, find
        all the InstructionFormat objects and set their name from the dict key.
        This is used to name a bunch of global variables in a module.
        """
        for name, obj in globs.items():
            if isinstance(obj, InstructionFormat):
                assert obj.name is None
                obj.name = name

class FormatField(object):
    """
    A field in an instruction format.

    This corresponds to a single member of a variant of the `InstructionData`
    data type.

    :param format: Parent `InstructionFormat`.
    :param operand: Operand number in parent.
    :param name: Member name in `InstructionData` variant.
    """

    def __init__(self, format, operand, name):
        # type: (InstructionFormat, int, str) -> None
        self.format = format
        self.operand = operand
        self.name = name

    def __str__(self):
        # type: () -> str
        return '{}.{}'.format(self.format.name, self.name)

    def rust_name(self):
        # type: () -> str
        if self.format.boxed_storage:
            return 'data.' + self.name
        else:
            return self.name
<commit_msg>Eliminate InstructionFormat.members.<commit_after>"""Classes for describing instruction formats."""
from __future__ import absolute_import
from .operands import OperandKind, VALUE, VARIABLE_ARGS
from .operands import Operand  # noqa

# The typing module is only required by mypy, and we don't use these imports
# outside type comments.
try:
    from typing import Dict, List, Tuple, Union, Any, Sequence, Iterable  # noqa
except ImportError:
    pass

class InstructionFormat(object):
    """
    Every instruction opcode has a corresponding instruction format which
    determines the number of operands and their kinds. Instruction formats are
    identified structurally, i.e., the format of an instruction is derived from
    the kinds of operands used in its declaration.

    Most instruction formats produce a single result, or no result at all. If
    an instruction can produce more than one result, the `multiple_results`
    flag must be set on its format. All results are of the `value` kind, and
    the instruction format does not keep track of how many results are
    produced. Some instructions, like `call`, may have a variable number of
    results.

    All instruction formats must be predefined in the
    :py:mod:`cretonne.formats` module.

    :param kinds: List of `OperandKind` objects describing the operands.
    :param name: Instruction format name in CamelCase. This is used as a Rust
        variant name in both the `InstructionData` and `InstructionFormat`
        enums.
    :param multiple_results: Set to `True` if this instruction format allows
        more than one result to be produced.
    :param value_list: Set to `True` if this instruction format uses a
        `ValueList` member to store its value operands.
    :param boxed_storage: Set to `True` is this instruction format requires a
        `data: Box<...>` pointer to additional storage in its `InstructionData`
        variant.
    :param typevar_operand: Index of the input operand that is used to infer
        the controlling type variable. By default, this is the first `value`
        operand.
    """

    # Map (multiple_results, kind, kind, ...) -> InstructionFormat
    _registry = dict()  # type: Dict[Tuple[bool, Tuple[OperandKind, ...]], InstructionFormat]  # noqa

    # All existing formats.
    all_formats = list()  # type: List[InstructionFormat]

    def __init__(self, *kinds, **kwargs):
        # type: (*Union[OperandKind, Tuple[str, OperandKind]], **Any) -> None # noqa
        self.name = kwargs.get('name', None)  # type: str
        self.multiple_results = kwargs.get('multiple_results', False)
        self.has_value_list = kwargs.get('value_list', False)
        self.boxed_storage = kwargs.get('boxed_storage', False)

        # Struct member names for the immediate operands. All other instruction
        # operands are values or variable argument lists. They are all handled
        # specially.
        self.imm_members = list()  # type: List[str]
        # Operand kinds for the immediate operands.
        self.imm_kinds = list()  # type: List[OperandKind]
        # The number of value operands stored in the format, or `None` when
        # `has_value_list` is set.
        self.num_value_operands = 0

        self.kinds = tuple(self._process_member_names(kinds))

        # Which of self.kinds are `value`?
        self.value_operands = tuple(
                i for i, k in enumerate(self.kinds) if k is VALUE)

        # The typevar_operand argument must point to a 'value' operand.
        self.typevar_operand = kwargs.get('typevar_operand', None)  # type: int
        if self.typevar_operand is not None:
            assert self.kinds[self.typevar_operand] is VALUE, \
                    "typevar_operand must indicate a 'value' operand"
        elif len(self.value_operands) > 0:
            # Default to the first 'value' operand, if there is one.
            self.typevar_operand = self.value_operands[0]

        # Compute a signature for the global registry.
        sig = (self.multiple_results, self.kinds)
        if sig in InstructionFormat._registry:
            raise RuntimeError(
                "Format '{}' has the same signature as existing format '{}'"
                .format(self.name, InstructionFormat._registry[sig]))
        InstructionFormat._registry[sig] = self
        InstructionFormat.all_formats.append(self)

    def _process_member_names(self, kinds):
        # type: (Sequence[Union[OperandKind, Tuple[str, OperandKind]]]) -> Iterable[OperandKind] # noqa
        """
        Extract names of all the immediate operands in the kinds tuple.

        Each entry is either an `OperandKind` instance, or a `(member, kind)`
        pair. The member names correspond to members in the Rust
        `InstructionData` data structure.

        Update the fields `num_value_operands`, `imm_kinds`, and `imm_members`.

        Yields the operand kinds.
        """
        for arg in kinds:
            if isinstance(arg, OperandKind):
                member = arg.default_member
                k = arg
            else:
                member, k = arg

            # We define 'immediate' as not a value or variable arguments.
            if k is VALUE:
                self.num_value_operands += 1
            elif k is VARIABLE_ARGS:
                # We require a value list for storage of variable arguments.
                assert self.has_value_list, "Need a value list"
            else:
                self.imm_kinds.append(k)
                self.imm_members.append(member)

            yield k

    def __str__(self):
        # type: () -> str
        args = ', '.join('{}: {}'.format(m, k)
                         for m, k in zip(self.imm_members, self.imm_kinds))
        return '{}({}, values={})'.format(
                self.name, args, self.num_value_operands)

    def __getattr__(self, attr):
        # type: (str) -> FormatField
        """
        Make immediate instruction format members available as attributes.

        Each non-value format member becomes a corresponding `FormatField`
        attribute.
        """
        try:
            i = self.imm_members.index(attr)
        except ValueError:
            raise AttributeError(
                    '{} is neither a {} member or a '
                    .format(attr, self.name) +
                    'normal InstructionFormat attribute')
        field = FormatField(self, i, attr)
        setattr(self, attr, field)
        return field

    @staticmethod
    def lookup(ins, outs):
        # type: (Sequence[Operand], Sequence[Operand]) -> InstructionFormat
        """
        Find an existing instruction format that matches the given lists of
        instruction inputs and outputs.

        The `ins` and `outs` arguments correspond to the
        :py:class:`Instruction` arguments of the same name, except they must be
        tuples of :py:`Operand` objects.
        """
        if len(outs) == 1:
            multiple_results = outs[0].kind == VARIABLE_ARGS
        else:
            multiple_results = len(outs) > 1
        sig = (multiple_results, tuple(op.kind for op in ins))
        if sig not in InstructionFormat._registry:
            raise RuntimeError(
                    "No instruction format matches ins = ({}){}".format(
                        ", ".join(map(str, sig[1])),
                        "[multiple results]" if multiple_results else ""))
        return InstructionFormat._registry[sig]

    @staticmethod
    def extract_names(globs):
        """
        Given a dict mapping name -> object as returned by `globals()`, find
        all the InstructionFormat objects and set their name from the dict key.
        This is used to name a bunch of global variables in a module.
        """
        for name, obj in globs.items():
            if isinstance(obj, InstructionFormat):
                assert obj.name is None
                obj.name = name

class FormatField(object):
    """
    A field in an instruction format.

    This corresponds to a single member of a variant of the `InstructionData`
    data type.

    :param format: Parent `InstructionFormat`.
    :param operand: Immediate operand number in parent.
    :param name: Member name in `InstructionData` variant.
    """

    def __init__(self, format, operand, name):
        # type: (InstructionFormat, int, str) -> None
        self.format = format
        self.operand = operand
        self.name = name

    def __str__(self):
        # type: () -> str
        return '{}.{}'.format(self.format.name, self.name)

    def rust_name(self):
        # type: () -> str
        if self.format.boxed_storage:
            return 'data.' + self.name
        else:
            return self.name
<|endoftext|>


<commit_before># pylint: disable=C0103

""" ORGANIZATIONS API VIEWS """
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Avg, F

from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response

from api_manager.courseware_access import get_course_key
from api_manager.models import Organization
from api_manager.users.serializers import UserSerializer
from api_manager.utils import str2bool
from gradebook.models import StudentGradebook
from student.models import CourseEnrollment

from .serializers import OrganizationSerializer

class OrganizationsViewSet(viewsets.ModelViewSet):
    """
    Django Rest Framework ViewSet for the Organization model.
    """
    serializer_class = OrganizationSerializer
    model = Organization

    @action(methods=['get',])
    def metrics(self, request, pk):
        """
        Provide statistical information for the specified Organization
        """
        response_data = {}
        grade_avg = 0
        grade_complete_match_range = getattr(settings, 'GRADEBOOK_GRADE_COMPLETE_PROFORMA_MATCH_RANGE', 0.01)
        org_user_grades = StudentGradebook.objects.filter(user__organizations=pk, user__is_active=True)
        courses_filter = request.QUERY_PARAMS.get('courses', None)
        if courses_filter:
            upper_bound = getattr(settings, 'API_LOOKUP_UPPER_BOUND', 100)
            courses_filter = courses_filter.split(",")[:upper_bound]
            courses = []
            for course_string in courses_filter:
                courses.append(get_course_key(course_string))
            org_user_grades = org_user_grades.filter(course_id__in=courses)

        users_grade_average = org_user_grades.aggregate(Avg('grade'))
        if users_grade_average['grade__avg']:
            grade_avg = float('{0:.3f}'.format(float(users_grade_average['grade__avg'])))
        response_data['users_grade_average'] = grade_avg

        users_grade_complete_count = org_user_grades\
            .filter(proforma_grade__lte=F('grade') + grade_complete_match_range, proforma_grade__gt=0).count()
        response_data['users_grade_complete_count'] = users_grade_complete_count

        return Response(response_data, status=status.HTTP_200_OK)

    @action(methods=['get', 'post'])
    def users(self, request, pk):
        """
        Add a User to an Organization
        """
        if request.method == 'GET':
            include_course_counts = request.QUERY_PARAMS.get('include_course_counts', None)
            users = User.objects.filter(organizations=pk)
            response_data = []
            if users:
                for user in users:
                    serializer = UserSerializer(user)
                    user_data = serializer.data
                    if str2bool(include_course_counts):
                        enrollments = CourseEnrollment.enrollments_for_user(user).count()
                        user_data['course_count'] = enrollments
                    response_data.append(user_data)
            return Response(response_data, status=status.HTTP_200_OK)
        else:
            user_id = request.DATA.get('id')
            try:
                user = User.objects.get(id=user_id)
            except ObjectDoesNotExist:
                message = 'User {} does not exist'.format(user_id)
                return Response({"detail": message}, status.HTTP_400_BAD_REQUEST)
            organization = self.get_object()
            organization.users.add(user)
            organization.save()
            return Response({}, status=status.HTTP_201_CREATED)
<commit_msg>added active flag on organisation metrics<commit_after># pylint: disable=C0103

""" ORGANIZATIONS API VIEWS """
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Avg, F

from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response

from api_manager.courseware_access import get_course_key
from api_manager.models import Organization
from api_manager.users.serializers import UserSerializer
from api_manager.utils import str2bool
from gradebook.models import StudentGradebook
from student.models import CourseEnrollment

from .serializers import OrganizationSerializer

class OrganizationsViewSet(viewsets.ModelViewSet):
    """
    Django Rest Framework ViewSet for the Organization model.
    """
    serializer_class = OrganizationSerializer
    model = Organization

    @action(methods=['get',])
    def metrics(self, request, pk):
        """
        Provide statistical information for the specified Organization
        """
        response_data = {}
        grade_avg = 0
        grade_complete_match_range = getattr(settings, 'GRADEBOOK_GRADE_COMPLETE_PROFORMA_MATCH_RANGE', 0.01)
        org_user_grades = StudentGradebook.objects.filter(user__organizations=pk, user__is_active=True,
                                                          user__courseenrollment__is_active=True)
        courses_filter = request.QUERY_PARAMS.get('courses', None)
        if courses_filter:
            upper_bound = getattr(settings, 'API_LOOKUP_UPPER_BOUND', 100)
            courses_filter = courses_filter.split(",")[:upper_bound]
            courses = []
            for course_string in courses_filter:
                courses.append(get_course_key(course_string))
            org_user_grades = org_user_grades.filter(course_id__in=courses)

        users_grade_average = org_user_grades.aggregate(Avg('grade'))
        if users_grade_average['grade__avg']:
            grade_avg = float('{0:.3f}'.format(float(users_grade_average['grade__avg'])))
        response_data['users_grade_average'] = grade_avg

        users_grade_complete_count = org_user_grades\
            .filter(proforma_grade__lte=F('grade') + grade_complete_match_range, proforma_grade__gt=0).count()
        response_data['users_grade_complete_count'] = users_grade_complete_count

        return Response(response_data, status=status.HTTP_200_OK)

    @action(methods=['get', 'post'])
    def users(self, request, pk):
        """
        Add a User to an Organization
        """
        if request.method == 'GET':
            include_course_counts = request.QUERY_PARAMS.get('include_course_counts', None)
            users = User.objects.filter(organizations=pk)
            response_data = []
            if users:
                for user in users:
                    serializer = UserSerializer(user)
                    user_data = serializer.data
                    if str2bool(include_course_counts):
                        enrollments = CourseEnrollment.enrollments_for_user(user).count()
                        user_data['course_count'] = enrollments
                    response_data.append(user_data)
            return Response(response_data, status=status.HTTP_200_OK)
        else:
            user_id = request.DATA.get('id')
            try:
                user = User.objects.get(id=user_id)
            except ObjectDoesNotExist:
                message = 'User {} does not exist'.format(user_id)
                return Response({"detail": message}, status.HTTP_400_BAD_REQUEST)
            organization = self.get_object()
            organization.users.add(user)
            organization.save()
            return Response({}, status=status.HTTP_201_CREATED)
<|endoftext|>


<commit_before>import datetime
from urlparse import urljoin
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.db.models import get_model, Count
from locksmith.common import apicall

APISTATS_APP = 'api'
APISTATS_MODEL = 'LogEntry'
APISTATS_DATE_FIELD = 'timestamp'
APISTATS_ENDPOINT_FIELD = 'method'
APISTATS_USER_FIELD = 'caller_key'

class Command(BaseCommand):
    help = "Push a given day's logs up to the analytics hub"
    args = '[date]'

    requires_model_validation = False

    def handle(self, date=None, *args, **options):
        if date:
            begin = datetime.datetime.strptime(date, '%Y-%m-%d')
        else:
            begin = datetime.datetime.now() - datetime.timedelta(days=1)
            date = begin.strftime('%Y-%m-%d')

        end = begin + datetime.timedelta(days=1)

        timestamp_fieldname = '%s__range' % APISTATS_DATE_FIELD

        Model = get_model(APISTATS_APP, APISTATS_MODEL)
        qs = Model.objects.filter(**{timestamp_fieldname : (begin, end)})
        results = qs.values(APISTATS_ENDPOINT_FIELD, APISTATS_USER_FIELD).annotate(calls=Count('id'))

        endpoint = urljoin(settings.LOCKSMITH_HUB_URL, '/report_views/')

        for item in results:
            data = {'api': settings.LOCKSMITH_API_NAME, 'date':date,
                    'endpoint': item[APISTATS_ENDPOINT_FIELD],
                    'key': item[APISTATS_USER_FIELD], 'calls': item['calls']}
            apicall(endpoint, settings.LOCKSMITH_SIGNING_KEY,
                    api=settings.LOCKSMITH_API_NAME, date=date,
                    endpoint=item[APISTATS_ENDPOINT_FIELD],
                    key=item[APISTATS_USER_FIELD], calls=item['calls'])
<commit_msg>LOCKSMITH_STATS_ settings<commit_after>import datetime
from urlparse import urljoin
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.db.models import get_model, Count
from locksmith.common import apicall

APP = getattr(settings, 'LOCKSMITH_STATS_APP', 'api')
MODEL = getattr(settings, 'LOCKSMITH_STATS_MODEL', 'LogEntry')
DATE_FIELD = getattr(settings, 'LOCKSMITH_STATS_DATE_FIELD', 'timestamp')
ENDPOINT_FIELD = getattr(settings, 'LOCKSMITH_STATS_ENDPOINT_FIELD', 'method')
USER_FIELD = getattr(settings, 'LOCKSMITH_STATS_USER_FIELD', 'caller_key')

LogModel = get_model(APP, MODEL)

class Command(BaseCommand):
    help = "Push a given day's logs up to the analytics hub"
    args = '[date]'
    requires_model_validation = False

    def handle(self, date=None, *args, **options):
        # calculate begin & end
        if date:
            begin = datetime.datetime.strptime(date, '%Y-%m-%d')
        else:
            begin = datetime.datetime.now() - datetime.timedelta(days=1)
            date = begin.strftime('%Y-%m-%d')
        end = begin + datetime.timedelta(days=1)

        # construct database query
        timestamp_fieldname = '%s__range' % DATE_FIELD
        qs = LogModel.objects.filter(**{timestamp_fieldname : (begin, end)})
        results = qs.values(ENDPOINT_FIELD, USER_FIELD).annotate(calls=Count('id'))

        endpoint = urljoin(settings.LOCKSMITH_HUB_URL, 'report_views/')

        # report results
        for item in results:
            apicall(endpoint, settings.LOCKSMITH_SIGNING_KEY,
                    api=settings.LOCKSMITH_API_NAME, date=date,
                    endpoint=item[ENDPOINT_FIELD],
                    key=item[USER_FIELD], calls=item['calls'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
from hbasetable import HBaseTable

class UidTable(HBaseTable):
	def table(self):
		return "uid"

	def create(self):
		return """
		{NAME => 'name', VERSIONS => 1,COMPRESSION => 'NONE'},
		{NAME => 'id',   VERSIONS => 1,COMPRESSION => 'NONE'}
		"""

	def validate(self, schema):
		return True

class DimensionHourTable(HBaseTable):
	def table(self):
		return "dimension-hour"

	def create(self):
		return """
		{NAME => 'md',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		if not("'ml'" in schema):
			self.alter("{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		if not("'mc'" in schema):
			self.alter("{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		return True

class DimensionDayTable(HBaseTable):
	def table(self):
		return "dimension-day"

	def create(self):
		return """
		{NAME => 'tc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'tl', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'h',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'hc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class VisitorTable(HBaseTable):
	def table(self):
		return "visitor"

	def create(self):
		return """
		{NAME => 'i', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 's', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class SessionTable(HBaseTable):
	def table(self):
		return "session"

	def create(self):
		return """
		{NAME => 'hp', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'i',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'p',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class DcsidLookupTable(HBaseTable):
	def table(self):
		return "dcsid-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class DimensionLookupTable(HBaseTable):
	def table(self):
		return "dimension-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class NextIdTable(HBaseTable):
	def table(self):
		return "next-id"

	def create(self):
		return """
		{NAME => 'id', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

HBaseTable.manage(UidTable())
HBaseTable.manage(DimensionHourTable())
HBaseTable.manage(DimensionDayTable())
HBaseTable.manage(VisitorTable())
HBaseTable.manage(SessionTable())
HBaseTable.manage(DcsidLookupTable())
HBaseTable.manage(DimensionLookupTable())
HBaseTable.manage(NextIdTable())

if (len(sys.argv) != 3):
	print "usage:   ./create_tables.py <dc> <pod>"
	print "example: ./create_tables.py pdx V"
	sys.exit(-1)

HBaseTable.dc = sys.argv[1]
HBaseTable.pod = sys.argv[2]

print "dc: " + HBaseTable.dc
print "pod: " + HBaseTable.pod

sys.exit(0 if HBaseTable.validate() else -1)

<commit_msg>adding metadata schema<commit_after>#!/usr/bin/env python

import sys
from hbasetable import HBaseTable

class UidTable(HBaseTable):
	def table(self):
		return "uid"

	def create(self):
		return """
		{NAME => 'name', VERSIONS => 1,COMPRESSION => 'NONE'},
		{NAME => 'id',   VERSIONS => 1,COMPRESSION => 'NONE'}
		"""

	def validate(self, schema):
		return True

class DimensionHourTable(HBaseTable):
	def table(self):
		return "dimension-hour"

	def create(self):
		return """
		{NAME => 'md',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		if not("'ml'" in schema):
			self.alter("{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		if not("'mc'" in schema):
			self.alter("{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		return True

class DimensionDayTable(HBaseTable):
	def table(self):
		return "dimension-day"

	def create(self):
		return """
		{NAME => 'tc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'tl', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'h',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'hc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class VisitorTable(HBaseTable):
	def table(self):
		return "visitor"

	def create(self):
		return """
		{NAME => 'i', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 's', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class SessionTable(HBaseTable):
	def table(self):
		return "session"

	def create(self):
		return """
		{NAME => 'hp', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'i',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'p',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class DcsidLookupTable(HBaseTable):
	def table(self):
		return "dcsid-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class DimensionLookupTable(HBaseTable):
	def table(self):
		return "dimension-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class NextIdTable(HBaseTable):
	def table(self):
		return "next-id"

	def create(self):
		return """
		{NAME => 'id', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True

class MetadataTable(HBaseTable):
	def table(self):
		return "metadata"

	def create(self):
		return """
		{NAME=>'meta'}
		"""

	def validate(self, schema):
		return True

HBaseTable.manage(UidTable())
HBaseTable.manage(DimensionHourTable())
HBaseTable.manage(DimensionDayTable())
HBaseTable.manage(VisitorTable())
HBaseTable.manage(SessionTable())
HBaseTable.manage(DcsidLookupTable())
HBaseTable.manage(DimensionLookupTable())
HBaseTable.manage(NextIdTable())
HBaseTable.manage(MetadataTable())

if (len(sys.argv) != 3):
	print "usage:   ./create_tables.py <dc> <pod>"
	print "example: ./create_tables.py pdx V"
	sys.exit(-1)

HBaseTable.dc = sys.argv[1]
HBaseTable.pod = sys.argv[2]

print "dc: " + HBaseTable.dc
print "pod: " + HBaseTable.pod

sys.exit(0 if HBaseTable.validate() else -1)

<|endoftext|>


<commit_before>import math

import numpy as np

from numerical_differentiation import gradient_approximation, \
                                      Hessian_approximation
from line_search import line_search_by_backtracking
from utils import compute_error

def gradient_descent(f, x_0, tol,
                     tol_backtracking, x_ast=None, p_ast=None, maxiter=30,
                     gf_symbolic=None):
    '''
    Method of gradient descent to numerically approximate solution of min f.
    Args:
        f (fun): definition of function f as lambda expression or function definition.
        x_0 (numpy ndarray): initial point for gradient descent method.
        tol (float): tolerance that will halt method. Controls norm of gradient of f.
        tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
        x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
        p_ast (float): value of f(x_ast), now it's required that user knows the solution...
        maxiter (int): maximum number of iterations.
        gf_symbolic (fun): definition of gradient of f. If given, no approximation is
                                     performed via finite differences.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
                                  of x_ast. Useful for plotting.
        x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
                                contains x, approximation of solution. Useful for plotting.
    '''
    iteration = 0
    x = x_0

    feval = f(x)
    if gf_symbolic:
        gfeval = gf_symbolic(x)
    else:
        gfeval = gradient_approximation(f,x)

    normgf = np.linalg.norm(gfeval)

    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,feval)

    Err = compute_error(x_ast,x)
    n = x.size
    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    print('I\tNormagf\t\tError x_ast\tError p_ast\tline search')
    print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}'.format(iteration,normgf,Err,Err_plot_aux[iteration],"---"))
    iteration+=1
    while(normgf>tol and iteration < maxiter):
        dir_desc = -gfeval
        der_direct = gfeval.dot(dir_desc)
        t = line_search_by_backtracking(f,dir_desc,x,der_direct)
        x = x + t*dir_desc
        feval = f(x)
        if gf_symbolic:
            gfeval = gf_symbolic(x)
        else:
            gfeval = gradient_approximation(f,x)
        normgf = np.linalg.norm(gfeval)
        Err_plot_aux[iteration]=compute_error(p_ast,feval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'.format(iteration,normgf,Err,
                                                                      Err_plot_aux[iteration],t))
        if t<tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    cond = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[cond]
    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
        x_plot = x_plot[:,:iteration]
    else:
        x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]

def Newtons_method(f, x_0, tol,
                   tol_backtracking, x_ast=None, p_ast=None, maxiter=30,
                   gf_symbolic=None,
                   Hf_symbolic=None):
    '''
    Newton's method to numerically approximate solution of min f.
    Args:
        f (fun): definition of function f as lambda expression or function definition.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method. Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
        x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
        p_ast (float): value of f(x_ast), now it's required that user knows the solution...
        maxiter (int): maximum number of iterations
        gf_symbolic (fun): definition of gradient of f. If given, no approximation is
                                     performed via finite differences.
        Hf_symbolic (fun): definition of Hessian of f. If given, no approximation is
                                     performed via finite differences.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
                          of x_ast. Useful for plotting.
        x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
                        contains x, approximation of solution. Useful for plotting.
    '''
    iteration = 0

    x = x_0

    feval = f(x)
    if gf_symbolic:
        gfeval = gf_symbolic(x)
    else:
        gfeval = gradient_approximation(f,x)

    if Hf_symbolic:
        Hfeval = Hf_symbolic(x)
    else:
        Hfeval = Hessian_approximation(f,x)

    normgf = np.linalg.norm(gfeval)
    condHf= np.linalg.cond(Hfeval)

    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,feval)

    Err = compute_error(x_ast,x)
    n = x.size
    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    #Newton's direction and Newton's decrement

    dir_Newton = np.linalg.solve(Hfeval, -gfeval)
    dec_Newton = -gfeval.dot(dir_Newton)

    print('I\tNormgf \tNewton Decrement\tError x_ast\tError p_ast\tline search\tCondHf')
    print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}\t\t{:0.2e}'.format(iteration,normgf,
                                                                         dec_Newton,Err,
                                                                         Err_plot_aux[iteration],"---",
                                                                         condHf))
    stopping_criteria = dec_Newton/2
    iteration+=1
    while(stopping_criteria>tol and iteration < maxiter):
        der_direct = -dec_Newton
        t = line_search_by_backtracking(f,dir_Newton,x,der_direct)
        x = x + t*dir_Newton
        feval = f(x)

        if gf_symbolic:
            gfeval = gf_symbolic(x)
        else:
            gfeval = gradient_approximation(f,x)

        if Hf_symbolic:
            Hfeval = Hf_symbolic(x)
        else:
            Hfeval = Hessian_approximation(f,x)

        normgf = np.linalg.norm(gfeval)
        condHf= np.linalg.cond(Hfeval)
        #Newton's direction and Newton's decrement

        dir_Newton = np.linalg.solve(Hfeval, -gfeval)
        dec_Newton = -gfeval.dot(dir_Newton)
        Err_plot_aux[iteration]=compute_error(p_ast,feval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'.format(iteration,normgf,
                                                                                dec_Newton,Err,
                                                                                Err_plot_aux[iteration],t,
                                                                                condHf))
        stopping_criteria = dec_Newton/2
        if t<tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    cond = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[cond]
    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
        x_plot = x_plot[:,:iteration]
    else:
        x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]<commit_msg>doing some refactor of lines<commit_after>import math

import numpy as np

from numerical_differentiation import gradient_approximation, \
                                      Hessian_approximation
from line_search import line_search_by_backtracking
from utils import compute_error

def gradient_descent(f, x_0, tol,
                     tol_backtracking, x_ast=None, p_ast=None, maxiter=30,
                     gf_symbolic=None):
    '''
    Method of gradient descent to numerically approximate solution of min f.
    Args:
        f (fun): definition of function f as lambda expression or function definition.
        x_0 (numpy ndarray): initial point for gradient descent method.
        tol (float): tolerance that will halt method. Controls norm of gradient of f.
        tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
        x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
        p_ast (float): value of f(x_ast), now it's required that user knows the solution...
        maxiter (int): maximum number of iterations.
        gf_symbolic (fun): definition of gradient of f. If given, no approximation is
                                     performed via finite differences.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
                                  of x_ast. Useful for plotting.
        x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
                                contains x, approximation of solution. Useful for plotting.
    '''
    iteration = 0
    x = x_0

    feval = f(x)
    if gf_symbolic:
        gfeval = gf_symbolic(x)
    else:
        gfeval = gradient_approximation(f,x)

    normgf = np.linalg.norm(gfeval)

    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,feval)

    Err = compute_error(x_ast,x)
    n = x.size
    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    print('I\tNormagf\t\tError x_ast\tError p_ast\tline search')
    print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}'.format(iteration,normgf,Err,Err_plot_aux[iteration],"---"))
    iteration+=1
    while(normgf>tol and iteration < maxiter):
        dir_desc = -gfeval
        der_direct = gfeval.dot(dir_desc)
        t = line_search_by_backtracking(f,dir_desc,x,der_direct)
        x = x + t*dir_desc
        feval = f(x)
        if gf_symbolic:
            gfeval = gf_symbolic(x)
        else:
            gfeval = gradient_approximation(f,x)
        normgf = np.linalg.norm(gfeval)
        Err_plot_aux[iteration]=compute_error(p_ast,feval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'.format(iteration,normgf,Err,
                                                                      Err_plot_aux[iteration],t))
        if t<tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    cond = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[cond]
    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
    else:
        if iteration == maxiter:
            print("Reached maximum of iterations, check approximation")
    x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]

def Newtons_method(f, x_0, tol,
                   tol_backtracking, x_ast=None, p_ast=None, maxiter=30,
                   gf_symbolic=None,
                   Hf_symbolic=None):
    '''
    Newton's method to numerically approximate solution of min f.
    Args:
        f (fun): definition of function f as lambda expression or function definition.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method. Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method. Controls value of line search by backtracking.
        x_ast (numpy ndarray): solution of min f, now it's required that user knows the solution...
        p_ast (float): value of f(x_ast), now it's required that user knows the solution...
        maxiter (int): maximum number of iterations
        gf_symbolic (fun): definition of gradient of f. If given, no approximation is
                                     performed via finite differences.
        Hf_symbolic (fun): definition of Hessian of f. If given, no approximation is
                                     performed via finite differences.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error between p_ast and f(x) with x approximation
                          of x_ast. Useful for plotting.
        x_plot (numpy ndarray): numpy array that containts in columns vector of approximations. Last column
                        contains x, approximation of solution. Useful for plotting.
    '''
    iteration = 0

    x = x_0

    feval = f(x)
    if gf_symbolic:
        gfeval = gf_symbolic(x)
    else:
        gfeval = gradient_approximation(f,x)

    if Hf_symbolic:
        Hfeval = Hf_symbolic(x)
    else:
        Hfeval = Hessian_approximation(f,x)

    normgf = np.linalg.norm(gfeval)
    condHf= np.linalg.cond(Hfeval)

    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,feval)

    Err = compute_error(x_ast,x)
    n = x.size
    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    #Newton's direction and Newton's decrement

    dir_Newton = np.linalg.solve(Hfeval, -gfeval)
    dec_Newton = -gfeval.dot(dir_Newton)

    print('I\tNormgf \tNewton Decrement\tError x_ast\tError p_ast\tline search\tCondHf')
    print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{}\t\t{:0.2e}'.format(iteration,normgf,
                                                                         dec_Newton,Err,
                                                                         Err_plot_aux[iteration],"---",
                                                                         condHf))
    stopping_criteria = dec_Newton/2
    iteration+=1
    while(stopping_criteria>tol and iteration < maxiter):
        der_direct = -dec_Newton
        t = line_search_by_backtracking(f,dir_Newton,x,der_direct)
        x = x + t*dir_Newton
        feval = f(x)

        if gf_symbolic:
            gfeval = gf_symbolic(x)
        else:
            gfeval = gradient_approximation(f,x)

        if Hf_symbolic:
            Hfeval = Hf_symbolic(x)
        else:
            Hfeval = Hessian_approximation(f,x)

        normgf = np.linalg.norm(gfeval)
        condHf= np.linalg.cond(Hfeval)
        #Newton's direction and Newton's decrement

        dir_Newton = np.linalg.solve(Hfeval, -gfeval)
        dec_Newton = -gfeval.dot(dir_Newton)
        Err_plot_aux[iteration]=compute_error(p_ast,feval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        print('{}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}\t{:0.2e}'.format(iteration,normgf,
                                                                                dec_Newton,Err,
                                                                                Err_plot_aux[iteration],t,
                                                                                condHf))
        stopping_criteria = dec_Newton/2
        if t<tol_backtracking: #if t is less than tol_backtracking then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    cond = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[cond]
    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
    else:
        if iteration == maxiter:
            print("Reached maximum of iterations, check approximation")
    x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]<|endoftext|>


<commit_before>#!/usr/bin/python
#Filename: endpoint.py
#Author: Bryce Drew , Stephen Kan
#Date: Sept. 29, 2015
#Functionality:
	#Functions for interacting with SPARQL endpoint. You can query and update the Db.
	#These should be called with SPARQL queries / updates as the data parameters, and the URI of your sparql endpoint as your URL.
	#See example functions below.
#Responsibilities:
	#The caller is responsible for sanitizing the input::These functions do NOT sanitize your input.
	#The caller is responsible for formatting the data into sparql queries
	#The caller is responsible for running a db server at the url provided.

import requests
import subprocess
import os
from SPARQLWrapper import SPARQLWrapper, JSON

#Change this to change where the sparql endpoint is pointing

#Takes a sparql query and a url of your sparql endpoint.
#Returns an object containing the information you requested. -- http://rdflib.readthedocs.org/en/latest/gettingstarted.html
def query(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql") ):
	sparql = SPARQLWrapper(url)
	sparql.setQuery(data)
	sparql.setReturnFormat(JSON)
	results = sparql.query().convert()
	return results

def ask(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql")):
	results = query(data)
	return results['boolean']

#This function uses the requests library, as it was already implemented when we switched to RDFLib (SPARQLWrapper). It uses 'update' instead of query.
#Update changes the graph, and you should sanitize your input to avoid vandalism.

def update(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql")):
	payload = {'update': data}
	r = requests.post(url, payload)
	return r.content

#Example data_location: "file:////home/ubiquitin/Documents/Ontologies/RDF Schemas for the Project/owl.ttl"
def file_update(data_location, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph")):
	payload = {'uri': data_location}
	r = requests.post(url, payload)
	return r

#These are example uses of these functions.

#Note: Should printing really be in this file? Probably not
'''
#Quick debugging print function. Takes the object returned from bgquery, and the name of all variables you want to display from your query. Unfortunatly, you have to provide the names.
def print_query(results,*args):
	for result in results["results"]["bindings"]:
		string = ""
		for i, thing in enumerate(args):
			string = string + result[thing]['value'] + " "
		print string
'''
'''
def print_spo(results):
	print_query(results,"s","p","o")
'''
<commit_msg>endpoint file update changed to blazegraph 2.0.0 friendly version<commit_after>#!/usr/bin/python
#Filename: endpoint.py
#Author: Bryce Drew , Stephen Kan
#Date: Sept. 29, 2015
#Functionality:
	#Functions for interacting with SPARQL endpoint. You can query and update the Db.
	#These should be called with SPARQL queries / updates as the data parameters, and the URI of your sparql endpoint as your URL.
	#See example functions below.
#Responsibilities:
	#The caller is responsible for sanitizing the input::These functions do NOT sanitize your input.
	#The caller is responsible for formatting the data into sparql queries
	#The caller is responsible for running a db server at the url provided.

import requests
import subprocess
import os
from SPARQLWrapper import SPARQLWrapper, JSON

#Change this to change where the sparql endpoint is pointing

#Takes a sparql query and a url of your sparql endpoint.
#Returns an object containing the information you requested. -- http://rdflib.readthedocs.org/en/latest/gettingstarted.html
def query(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql") ):
	sparql = SPARQLWrapper(url)
	sparql.setQuery(data)
	sparql.setReturnFormat(JSON)
	results = sparql.query().convert()
	return results

def ask(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql")):
	results = query(data)
	return results['boolean']

#This function uses the requests library, as it was already implemented when we switched to RDFLib (SPARQLWrapper). It uses 'update' instead of query.
#Update changes the graph, and you should sanitize your input to avoid vandalism.

def update(data, url = os.getenv('SUPERPHY_RDF_URL', "http://localhost:9000/blazegraph/namespace/superphy/sparql")):
	payload = {'update': data}
	r = requests.post(url, payload)
	return r.content

def file_update(path):
	"""
		Will upload the file at given abolute path
	"""
	return update("""LOAD <file:%s>;""" % (path))
<|endoftext|>