<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo import fields, osv, models, api
from odoo.tools.translate import _
import logging
import meli_oerp_config

import logging
_logger = logging.getLogger(__name__)

from meli_oerp_config import *
from melisdk.meli import Meli

class mercadolibre_posting_update(models.TransientModel):
    _name = "mercadolibre.posting.update"
    _description = "Update Posting Questions"

    def posting_update(self, ids ):

        #posting_ids = context['active_ids']
        posting_ids = ids
        posting_obj = self.env['mercadolibre.posting']

        for posting_id in posting_ids:

            _logger.info("posting_update: %s " % (posting_id) )

            posting = posting_obj.browse(posting_id)
            posting_obj.posting_query_questions( posting_id )

        return {}

mercadolibre_posting_update()

class mercadolibre_posting(models.Model):
    _name = "mercadolibre.posting"
    _description = "Posting en MercadoLibre"

    def posting_update( self, ids, field_name, attributes, context=None):

        log_msg = 'posting_update: %s' % (field_name)
        _logger.info(log_msg)

        user_obj = self.pool.get('res.users').browse(cr, uid, uid)
        company = user_obj.company_id

        posting_obj = self.pool.get('mercadolibre.posting')
        posting = posting_obj.browse(cr, uid, ids[0])

        update_status = "ok"

        self.posting_query_questions( cr, uid, ids[0] )

        res = {}
        for posting in self.browse(cr,uid,ids):
            res[posting.id] = update_status
        return res

    def posting_query_questions( self, id ):

        #get with an item id
        company = self.env.user.company_id

        posting_obj = self.env['mercadolibre.posting']
        posting = posting_obj.browse( id )

        log_msg = 'posting_query_questions: %s' % (posting.meli_id)
        _logger.info(log_msg)

        CLIENT_ID = company.mercadolibre_client_id
        CLIENT_SECRET = company.mercadolibre_secret_key
        ACCESS_TOKEN = company.mercadolibre_access_token
        REFRESH_TOKEN = company.mercadolibre_refresh_token

        #
        meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN )

        response = meli.get("/items/"+posting.meli_id, {'access_token':meli.access_token})
        product_json = response.json()
        #_logger.info( product_json )

        if "error" in product_json:
            ML_status = product_json["error"]
        else:
            ML_status = product_json["status"]
            ML_permalink = product_json["permalink"]
            ML_price = product_json["price"]
            posting.write( { 'meli_status': ML_status, 'meli_permalink': ML_permalink, 'meli_price': ML_price } )

        response = meli.get("/questions/search?item_id="+posting.meli_id, {'access_token':meli.access_token})
        questions_json = response.json()
        #_logger.info( questions_json )

        questions_obj = self.env['mercadolibre.questions']

        if 'questions' in questions_json:
            questions = questions_json['questions']
            _logger.info( questions )
            cn = 0
            for Question in questions:
                cn = cn + 1
                _logger.info(cn)
                _logger.info(Question )

#{
#   'status': u'UNANSWERED',
#   'seller_id': 171319838,
#   'from': {
#      'answered_questions': 0,
#      'id': 171329758
#   },
#   'deleted_from_listing': False,
#   'text': 'Pregunta que hago para testear el m\xf3dulo de meli_oerp en Odoo....??',
#   'answer': None,
#   'item_id': u'MLA548679961',
#   'date_created': '2015-03-04T12:04:48.000-04:00',
#   'hold': False,
#   'id': 3471012014
#}

##{
#   'status': 'ACTIVE',
#   'text': 'Ok recibiose la pregunta correctamente, ahora viendo si sale esta respuesta! Saludos\nMas saludos',
#   'date_created': '2015-03-04T13:00:53.000-04:00'
#}

                question_answer = Question['answer']

                question_fields = {
                    'posting_id': posting.id,
                    'question_id': Question['id'],
                    'date_created': Question['date_created'],
                    'item_id': Question['item_id'],
                    'seller_id': Question['seller_id'],
                    'text': str(Question['text'].encode("utf-8")),
                    'status': Question['status'],
                }

                if (question_answer):
                    question_fields['answer_text'] = str(question_answer['text'].encode("utf-8"))
                    question_fields['answer_status'] = question_answer['status']
                    question_fields['answer_date_created'] = question_answer['date_created']

                question_fetch_ids = questions_obj.search( [('question_id','=',question_fields['question_id'])])

                if not question_fetch_ids:
	                question_fetch_ids = questions_obj.create( ( question_fields ))
                else:
                    questions_obj.write( question_fetch_ids[0], ( question_fields ) )

        return {}

    def posting_query_all_questions( self, cr, uid, ids, context=None ):

        return {}

    posting_date = fields.Date('Fecha del posting');
    name = fields.Char('Name');
    meli_id = fields.Char('Id del item asignado por Meli', size=256);
    product_id = fields.Many2one('product.product','product_id');
    meli_status = fields.Char( string="Estado del producto en MLA", size=256 );
    meli_permalink = fields.Char( string="Permalink en MercadoLibre", size=512 );
    meli_price = fields.Char(string='Precio de venta', size=128);
    posting_questions = fields.One2many( 'mercadolibre.questions','posting_id','Questions' );
    posting_update = fields.Char( compute=posting_update, string="Posting Update", store=False );

mercadolibre_posting()
<commit_msg>Product_update to 1 parameter<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo import fields, osv, models, api
from odoo.tools.translate import _
import logging
import meli_oerp_config

import logging
_logger = logging.getLogger(__name__)

from meli_oerp_config import *
from melisdk.meli import Meli

class mercadolibre_posting_update(models.TransientModel):
    _name = "mercadolibre.posting.update"
    _description = "Update Posting Questions"

    def posting_update(self, ids ):

        #posting_ids = context['active_ids']
        posting_ids = ids
        posting_obj = self.env['mercadolibre.posting']

        for posting_id in posting_ids:

            _logger.info("posting_update: %s " % (posting_id) )

            posting = posting_obj.browse(posting_id)
            posting_obj.posting_query_questions( posting_id )

        return {}

mercadolibre_posting_update()

class mercadolibre_posting(models.Model):
    _name = "mercadolibre.posting"
    _description = "Posting en MercadoLibre"

    def posting_update( self, ids):

        log_msg = 'posting_update: %s' % (field_name)
        _logger.info(log_msg)

        user_obj = self.pool.get('res.users').browse(cr, uid, uid)
        company = user_obj.company_id

        posting_obj = self.pool.get('mercadolibre.posting')
        posting = posting_obj.browse(cr, uid, ids[0])

        update_status = "ok"

        self.posting_query_questions( cr, uid, ids[0] )

        res = {}
        for posting in self.browse(cr,uid,ids):
            res[posting.id] = update_status
        return res

    def posting_query_questions( self, id ):

        #get with an item id
        company = self.env.user.company_id

        posting_obj = self.env['mercadolibre.posting']
        posting = posting_obj.browse( id )

        log_msg = 'posting_query_questions: %s' % (posting.meli_id)
        _logger.info(log_msg)

        CLIENT_ID = company.mercadolibre_client_id
        CLIENT_SECRET = company.mercadolibre_secret_key
        ACCESS_TOKEN = company.mercadolibre_access_token
        REFRESH_TOKEN = company.mercadolibre_refresh_token

        #
        meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN )

        response = meli.get("/items/"+posting.meli_id, {'access_token':meli.access_token})
        product_json = response.json()
        #_logger.info( product_json )

        if "error" in product_json:
            ML_status = product_json["error"]
        else:
            ML_status = product_json["status"]
            ML_permalink = product_json["permalink"]
            ML_price = product_json["price"]
            posting.write( { 'meli_status': ML_status, 'meli_permalink': ML_permalink, 'meli_price': ML_price } )

        response = meli.get("/questions/search?item_id="+posting.meli_id, {'access_token':meli.access_token})
        questions_json = response.json()
        #_logger.info( questions_json )

        questions_obj = self.env['mercadolibre.questions']

        if 'questions' in questions_json:
            questions = questions_json['questions']
            _logger.info( questions )
            cn = 0
            for Question in questions:
                cn = cn + 1
                _logger.info(cn)
                _logger.info(Question )

#{
#   'status': u'UNANSWERED',
#   'seller_id': 171319838,
#   'from': {
#      'answered_questions': 0,
#      'id': 171329758
#   },
#   'deleted_from_listing': False,
#   'text': 'Pregunta que hago para testear el m\xf3dulo de meli_oerp en Odoo....??',
#   'answer': None,
#   'item_id': u'MLA548679961',
#   'date_created': '2015-03-04T12:04:48.000-04:00',
#   'hold': False,
#   'id': 3471012014
#}

##{
#   'status': 'ACTIVE',
#   'text': 'Ok recibiose la pregunta correctamente, ahora viendo si sale esta respuesta! Saludos\nMas saludos',
#   'date_created': '2015-03-04T13:00:53.000-04:00'
#}

                question_answer = Question['answer']

                question_fields = {
                    'posting_id': posting.id,
                    'question_id': Question['id'],
                    'date_created': Question['date_created'],
                    'item_id': Question['item_id'],
                    'seller_id': Question['seller_id'],
                    'text': str(Question['text'].encode("utf-8")),
                    'status': Question['status'],
                }

                if (question_answer):
                    question_fields['answer_text'] = str(question_answer['text'].encode("utf-8"))
                    question_fields['answer_status'] = question_answer['status']
                    question_fields['answer_date_created'] = question_answer['date_created']

                question_fetch_ids = questions_obj.search( [('question_id','=',question_fields['question_id'])])

                if not question_fetch_ids:
	                question_fetch_ids = questions_obj.create( ( question_fields ))
                else:
                    questions_obj.write( question_fetch_ids[0], ( question_fields ) )

        return {}

    def posting_query_all_questions( self, cr, uid, ids, context=None ):

        return {}

    posting_date = fields.Date('Fecha del posting');
    name = fields.Char('Name');
    meli_id = fields.Char('Id del item asignado por Meli', size=256);
    product_id = fields.Many2one('product.product','product_id');
    meli_status = fields.Char( string="Estado del producto en MLA", size=256 );
    meli_permalink = fields.Char( string="Permalink en MercadoLibre", size=512 );
    meli_price = fields.Char(string='Precio de venta', size=128);
    posting_questions = fields.One2many( 'mercadolibre.questions','posting_id','Questions' );
    posting_update = fields.Char( compute=posting_update, string="Posting Update", store=False );

mercadolibre_posting()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Feb 22 22:09:50 2015

@author: mcurrie

PayPal API reference:
https://developer.paypal.com/docs/api/

TicketLeap API reference:
http://dev.ticketleap.com/
(for now the information is not detailed enough for use for transactions)

"""

import petl as etl
import csv

import sys, os, datetime, time

def main(argv):
    """
    INPUT: paypal.csv
    OUTPUT: output.iif and unprocessed.csv

    """
    # --------------------
    # 1. HARDCODED SETUP INFO
    etl.config.look_style = 'minimal'
    input_folder = 'C:\\Users\\mcurrie\\Desktop\\GitHub\\TicketLeapToQuickBooks'
    input_pp = os.path.join(input_folder, 'paypal.csv')
    start_date = datetime.date(2015, 1, 1)
    end_date = None
    output_iif = os.path.join(input_folder, 'output.iif')
    output_unprocessed = os.path.join(input_folder, 'unprocessed.csv')

    # TODO: Validate that the cart_payment is associated with exactly
    #       cart_payment.Quantity cart_items.

    # --------------------
    # 2. LOAD PAYPAL CSV FILE
    print("Loading PayPal input file")
    paypal = etl.fromcsv(input_pp)
    paypal = cleanup_paypal(paypal)

    # DEBUG: TO NARROW THINGS FOR NOW
    paypal = paypal.selecteq('Name', 'Alex Rossol')

    # --------------------
    # 3. CREATE QUICKBOOKS IIF FILE
    print("Creating output IIF file")
    output_iif_file = get_output_file(output_iif)

    output_iif_file.write(handle_sales_receipts(paypal))
    output_iif_file.write(handle_invoices(paypal))
    output_iif_file.write(handle_ticketleap_fees(paypal))
    output_iif_file.write(handle_customer_names(paypal))

    output_iif_file.close()

    # --------------------
    # 4. CREATE UNPROCESSED ROWS FILE
    print("Creating output unprocessed rows CSV file")

    output_unprocessed_file = get_output_file(output_unprocessed)
    writer = csv.writer(output_unprocessed_file, lineterminator='\n')

    # TODO: add ONLY the unprocessed rows to output_unprocessed_file
    writer.writerows(paypal)
    output_unprocessed_file.close()

def handle_sales_receipts(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the sales receipts (in a string)

    """
    out_string = ""

    out_string += '!TRNS	DATE	ACCNT	NAME	CLASS	AMOUNT	MEMO\n'
    out_string += '!SPL	DATE	ACCNT	NAME	AMOUNT	MEMO\n'
    out_string += '!ENDTRNS\n'

    # Sales receipts are organized in the CSV file as a row to summarize,
    # (cart payment), plus one or more rows for each of the items purchased.
    cart_payments = paypal.selecteq('Type', 'Shopping Cart Payment Received')
    # Ignore refunds
    cart_payments = cart_payments.selecteq('Status', 'Completed')
    cart_payments = cart_payments.cut('Date', 'Name', 'Email', 'Gross', 'Fee',
                                      'Transaction ID')

    cart_items = paypal.selecteq('Type', 'Shopping Cart Item')
    cart_items = cart_items.cut('Item Title', 'Item ID', 'Quantity', 'Gross',
                                'Transaction ID')

    # TODO ABORT IF EMPTY

    for tranID in cart_payments.columns()['Transaction ID']:
        cur_cart_payment = cart_payments.selecteq('Transaction ID', tranID)
        cur_cart_items = cart_items.selecteq('Transaction ID', tranID).columns()

        # TODO ABORT IF EMPTY

        # TODO WRITE TO IIF
        pass

        for item_number in range(len(cur_cart_items)):
            # TODO WRITE TO IIF
            pass

    # TODO WRITE CLOSING STATEMENT TO IIF

    return out_string

def handle_invoices(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the invoices and payments received for them

    """
    out_string = ""
    # TODO
    pass
    return out_string

def handle_ticketleap_fees(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the ticketleap payments (the fees they charge us for using TicketLeap)

    """
    out_string = ""
    # TODO
    pass
    return out_string

def handle_customer_names(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the unique customer names

    """
    out_string = ""

    names = paypal.cut('Name', 'Email', 'Address Line 1', 'Address Line 2',
                       'Town/City', 'Province', 'Postal Code', 'Country',
                       'Phone')
    # TODO: ensure that 'NAME' doesn't already exist in QuickBooks? How does
    # or maybe just prevent that vendor/customer error by appending '(c)'
    # after every name
    names_dict = names.columns()
    # Returns rows with same Name but differing in some other field
    name_conflicts = names.conflicts('Name')

    names = names.addfield('First Name', lambda rec: rec['Name'].split()[0])
    names = names.addfield('Last Name',  lambda rec: rec['Name'].split()[1])
    names = names.addfield('Taxable', 'N')
    names = names.addfield('Address1', lambda rec: rec['Address Line 1'] +
                                           ' ' + rec['Address Line 2'])
    names = names.addfield('Address2', lambda rec: rec['Town/City'] + ', ' +
                                           rec['Province'] + ' ' +
                                           rec['Postal Code'])

    names_header = ['!CUST', 'NAME', 'BADDR1', 'BADDR2', 'BADDR3',
                    'BADDR4', 'BADDR5', 'SADDR1', 'SADDR2', 'SADDR3',
                    'SADDR4', 'SADDR5', 'PHONE1', 'PHONE2', 'FAXNUM',
                    'EMAIL', 'NOTE', 'CONT1', 'CONT2', 'CTYPE', 'TERMS',
                    'TAXABLE', 'LIMIT', 'RESALENUM', 'REP', 'TAXITEM',
                    'NOTEPAD', 'SALUTATION', 'COMPANYNAME', 'FIRSTNAME',
                    'MIDINIT', 'LASTNAME']

    """
    {'NAME': 'Name',
     'BADDR1': 'Name',
     'BADDR2': 'Address1',
     'BADDR3': 'Address2',
     'BADDR4': 'Country',
     'TAXABLE': 'Taxable',
     'EMAIL': 'Email',
     'PHONE1': 'Phone Number',
     'FIRSTNAME': 'First Name',
     'LASTNAME': 'Last Name'}
    """

    return out_string

def cleanup_paypal(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and clean it up

    """
    # Remove leading and trailing whitespace in the header row
    paypal_clean = paypal.setheader(list((x.strip() for x in paypal.header())))

    # Rename some header names to more convenient names
    paypal_clean = paypal_clean.rename('From Email Address', 'Email')
    paypal_clean = paypal_clean.rename('Contact Phone Number', 'Phone')
    paypal_clean = paypal_clean.rename('Zip/Postal Code', 'Postal Code')
    paypal_clean = paypal_clean.rename(
        'State/Province/Region/County/Territory/Prefecture/Republic',
        'Province')
    paypal_clean = paypal_clean.rename(
        'Address Line 2/District/Neighborhood',
        'Address Line 2')

    # It seems that Quickbooks requires that phone numbers be entirely
    # composed of digits, e.g. 4035559195 instead of 403-555-9195
    paypal_clean = paypal_clean.convert('Phone',
                       lambda v: ''.join(c for c in v if c in digits))

    # Get rid of columns that are not needed
    paypal_clean = paypal_clean.cutout(
        'Shipping and Handling Amount', 'Insurance Amount', 'Sales Tax',
        'Option 1 Name', 'Option 1 Value', 'Option 2 Name', 'Option 2 Value',
        'Auction Site', 'Buyer ID', 'Item URL', 'Closing Date', 'Escrow Id',
        'Invoice Id', 'Time Zone', 'Net', 'Balance', 'Counterparty Status',
        'Address Status')

    # Convert the 'Date' column to proper Python datetime-type dates
    def convert_ppdate(paypal_date):
        ex_struct_time = time.strptime(paypal_date, '%m/%d/%Y')
        ex_date = datetime.date(ex_struct_time.tm_year,
                                ex_struct_time.tm_mon,
                                ex_struct_time.tm_mday)
        return ex_date

    paypal_clean = etl.transform.conversions.convert(paypal_clean,
                                                     'Date',
                                                     convert_ppdate)

    # Convert the money and quantity columns from string to float
    paypal_clean = etl.transform.conversions.convert(paypal_clean,
                                                     ('Gross', 'Fee',
                                                      'Quantity'),
                                                     float)
    return paypal_clean

def get_output_file(file_name):
    try:
        file = open(file_name, 'w')
    except:
        print("Could not open " + file_name + " for writing")
        sys.exit(0)

    return file

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>Added output improvement outline<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Feb 22 22:09:50 2015

@author: mcurrie

PayPal API reference:
https://developer.paypal.com/docs/api/

TicketLeap API reference:
http://dev.ticketleap.com/
(for now the information is not detailed enough for use for transactions)

"""

import petl as etl
import csv

import sys, os, datetime, time

def main(argv):
    """
    INPUT: paypal.csv
    OUTPUT: output.iif and unprocessed.csv

    """
    # --------------------
    # 1. HARDCODED SETUP INFO
    etl.config.look_style = 'minimal'
    input_folder = 'C:\\Users\\mcurrie\\Desktop\\GitHub\\TicketLeapToQuickBooks'
    input_pp = os.path.join(input_folder, 'paypal.csv')
    start_date = datetime.date(2015, 1, 1)
    end_date = None
    output_iif = os.path.join(input_folder, 'output.iif')
    output_unprocessed = os.path.join(input_folder, 'unprocessed.csv')

    # TODO: Validate that the cart_payment is associated with exactly
    #       cart_payment.Quantity cart_items.

    # --------------------
    # 2. LOAD PAYPAL CSV FILE
    print("Loading PayPal input file")
    paypal = etl.fromcsv(input_pp)
    paypal = cleanup_paypal(paypal)

    # DEBUG: TO NARROW THINGS FOR NOW
    paypal = paypal.selecteq('Name', 'Alex Rossol')

    # --------------------
    # 3. CREATE QUICKBOOKS IIF FILE
    print("Creating output IIF file")
    output_iif_file = get_output_file(output_iif)

    output_iif_file.write(handle_sales_receipts(paypal))
    output_iif_file.write(handle_invoices(paypal))
    output_iif_file.write(handle_ticketleap_fees(paypal))
    output_iif_file.write(handle_customer_names(paypal))

    output_iif_file.close()

    # --------------------
    # 4. CREATE UNPROCESSED ROWS FILE
    print("Creating output unprocessed rows CSV file")

    output_unprocessed_file = get_output_file(output_unprocessed)
    writer = csv.writer(output_unprocessed_file, lineterminator='\n')

    # TODO: add ONLY the unprocessed rows to output_unprocessed_file
    writer.writerows(paypal)
    output_unprocessed_file.close()

def handle_sales_receipts(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the sales receipts (in a string)

    """
    out_string = ""

    out_string += '!TRNS	DATE	ACCNT	NAME	CLASS	AMOUNT	MEMO\n'
    out_string += '!SPL	DATE	ACCNT	NAME	AMOUNT	MEMO\n'
    out_string += '!ENDTRNS\n'

    # Sales receipts are organized in the CSV file as a row to summarize,
    # (cart payment), plus one or more rows for each of the items purchased.
    cart_payments = paypal.selecteq('Type', 'Shopping Cart Payment Received')
    # Ignore refunds
    cart_payments = cart_payments.selecteq('Status', 'Completed')
    cart_payments = cart_payments.cut('Date', 'Name', 'Email', 'Gross', 'Fee',
                                      'Transaction ID')

    cart_items = paypal.selecteq('Type', 'Shopping Cart Item')
    cart_items = cart_items.cut('Item Title', 'Item ID', 'Quantity', 'Gross',
                                'Transaction ID')

    # TODO ABORT IF EMPTY

    for tranID in cart_payments.columns()['Transaction ID']:
        cur_cart_payment = cart_payments.selecteq('Transaction ID', tranID)
        cur_cart_items = cart_items.selecteq('Transaction ID', tranID).columns()

        # TODO ABORT IF EMPTY

        # TODO WRITE TO IIF
        pass

        for item_number in range(len(cur_cart_items)):
            # TODO WRITE TO IIF
            pass

    # TODO WRITE CLOSING STATEMENT TO IIF

    return out_string

def handle_invoices(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the invoices and payments received for them

    """
    out_string = ""
    # TODO
    pass
    return out_string

def handle_ticketleap_fees(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the ticketleap payments (the fees they charge us for using TicketLeap)

    """
    out_string = ""
    # TODO
    pass
    return out_string

def handle_customer_names(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and spit out
    the unique customer names

    """
    out_string = ""

    names = paypal.cut('Name', 'Email', 'Address Line 1', 'Address Line 2',
                       'Town/City', 'Province', 'Postal Code', 'Country',
                       'Phone')
    # TODO: ensure that 'NAME' doesn't already exist in QuickBooks? How does
    # or maybe just prevent that vendor/customer error by appending '(c)'
    # after every name
    names_dict = names.columns()
    # Returns rows with same Name but differing in some other field
    name_conflicts = names.conflicts('Name')

    names = names.addfield('First Name', lambda rec: rec['Name'].split()[0])
    names = names.addfield('Last Name',  lambda rec: rec['Name'].split()[1])
    names = names.addfield('Taxable', 'N')
    names = names.addfield('Address1', lambda rec: rec['Address Line 1'] +
                                           ' ' + rec['Address Line 2'])
    names = names.addfield('Address2', lambda rec: rec['Town/City'] + ', ' +
                                           rec['Province'] + ' ' +
                                           rec['Postal Code'])

    names_header = ['!CUST', 'NAME', 'BADDR1', 'BADDR2', 'BADDR3',
                    'BADDR4', 'BADDR5', 'SADDR1', 'SADDR2', 'SADDR3',
                    'SADDR4', 'SADDR5', 'PHONE1', 'PHONE2', 'FAXNUM',
                    'EMAIL', 'NOTE', 'CONT1', 'CONT2', 'CTYPE', 'TERMS',
                    'TAXABLE', 'LIMIT', 'RESALENUM', 'REP', 'TAXITEM',
                    'NOTEPAD', 'SALUTATION', 'COMPANYNAME', 'FIRSTNAME',
                    'MIDINIT', 'LASTNAME']

    # TODO
    # you shouldn't even need out_string
    # you should be able to return a PETL table object that you can just
    # go table.appendtsv('output.iif') on!

    """
    {'NAME': 'Name',
     'BADDR1': 'Name',
     'BADDR2': 'Address1',
     'BADDR3': 'Address2',
     'BADDR4': 'Country',
     'TAXABLE': 'Taxable',
     'EMAIL': 'Email',
     'PHONE1': 'Phone Number',
     'FIRSTNAME': 'First Name',
     'LASTNAME': 'Last Name'}
    """

    return out_string

def cleanup_paypal(paypal):
    """
    Take a paypal csv file (already sucked into PETL) and clean it up

    """
    # Remove leading and trailing whitespace in the header row
    paypal_clean = paypal.setheader(list((x.strip() for x in paypal.header())))

    # Rename some header names to more convenient names
    paypal_clean = paypal_clean.rename('From Email Address', 'Email')
    paypal_clean = paypal_clean.rename('Contact Phone Number', 'Phone')
    paypal_clean = paypal_clean.rename('Zip/Postal Code', 'Postal Code')
    paypal_clean = paypal_clean.rename(
        'State/Province/Region/County/Territory/Prefecture/Republic',
        'Province')
    paypal_clean = paypal_clean.rename(
        'Address Line 2/District/Neighborhood',
        'Address Line 2')

    # It seems that Quickbooks requires that phone numbers be entirely
    # composed of digits, e.g. 4035559195 instead of 403-555-9195
    paypal_clean = paypal_clean.convert('Phone',
                       lambda v: ''.join(c for c in v if c in digits))

    # Get rid of columns that are not needed
    paypal_clean = paypal_clean.cutout(
        'Shipping and Handling Amount', 'Insurance Amount', 'Sales Tax',
        'Option 1 Name', 'Option 1 Value', 'Option 2 Name', 'Option 2 Value',
        'Auction Site', 'Buyer ID', 'Item URL', 'Closing Date', 'Escrow Id',
        'Invoice Id', 'Time Zone', 'Net', 'Balance', 'Counterparty Status',
        'Address Status')

    # Convert the 'Date' column to proper Python datetime-type dates
    def convert_ppdate(paypal_date):
        ex_struct_time = time.strptime(paypal_date, '%m/%d/%Y')
        ex_date = datetime.date(ex_struct_time.tm_year,
                                ex_struct_time.tm_mon,
                                ex_struct_time.tm_mday)
        return ex_date

    paypal_clean = etl.transform.conversions.convert(paypal_clean,
                                                     'Date',
                                                     convert_ppdate)

    # Convert the money and quantity columns from string to float
    paypal_clean = etl.transform.conversions.convert(paypal_clean,
                                                     ('Gross', 'Fee',
                                                      'Quantity'),
                                                     float)
    return paypal_clean

def get_output_file(file_name):
    try:
        file = open(file_name, 'w')
    except:
        print("Could not open " + file_name + " for writing")
        sys.exit(0)

    return file

if __name__ == "__main__":
    main(sys.argv[1:])<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import division

import os
import sys
from pdb import set_trace
from glob2 import glob

root = os.path.join(os.getcwd().split("src")[0], "src")
if root not in sys.path:
    sys.path.append(root)

def get_jar_paths():
    project_dir = os.path.join(root, "data/java/raw/")
    projects = glob(os.path.join(project_dir, "*"))
    path_dict = dict()
    for project in projects:
        files = glob(os.path.join(project, "**/*.jar"))
        for f in files:
            chunks = f.split("raw")[1].split("/")[1:]
            print(chunks[:2])
            try:
                path_dict[chunks[0]][chunks[1]].append(f)
            except Exception as e:
                if chunks[0] in e:
                    set_trace()
                    path_dict.update({
                        chunks[0]: {
                            chunks[1]: [f]
                        }})
                elif chunks[1] in e:
                    set_trace()
                    path_dict[chunks[0]].update({chunks[1]: [f]})

        set_trace()
    pass

if __name__ == "__main__":
    get_jar_paths()
<commit_msg>Lab Machine Changes<commit_after>from __future__ import print_function
from __future__ import division

import os
import sys
from pdb import set_trace
from glob2 import glob

root = os.path.join(os.getcwd().split("src")[0], "src")
if root not in sys.path:
    sys.path.append(root)

def get_jar_paths():
    project_dir = os.path.join(root, "data/java/raw/")
    projects = glob(os.path.join(project_dir, "*"))
    path_dict = dict()
    for project in projects:
        files = glob(os.path.join(project, "**/*.jar"))
        for f in files:
            chunks = f.split("raw")[1].split("/")[1:]
            print(chunks[:2])
            try:
                path_dict[chunks[0]][chunks[1]].append(f)
            except Exception as e:
                if chunks[0] in e:
                    path_dict.update({
                        chunks[0]: {
                            chunks[1]: [f]
                        }})
                elif chunks[1] in e:
                    path_dict[chunks[0]].update({chunks[1]: [f]})
        set_trace()
    pass

if __name__ == "__main__":
    get_jar_paths()
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.db import connections
from django.http import HttpResponse
from django.template.loader import render_to_string

from .forms import LoteForm

def rows_to_dict_list(cursor):
    columns = [i[0] for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def index(request):
    context = {}
    return render(request, 'lotes/index.html', context)

def posicao(request):
    context = {}
    if request.method == 'POST':
        form = LoteForm(request.POST)
        if form.is_valid():
            lote = form.cleaned_data['lote']
            periodo = lote[:4]
            ordem_confeccao = lote[-5:]
            cursor = connections['so'].cursor()
            sql = '''
                SELECT
                  l.CODIGO_ESTAGIO
                , e.DESCRICAO DESCRICAO_ESTAGIO
                , l.QTDE_EM_PRODUCAO_PACOTE
                FROM PCPC_040 l
                JOIN MQOP_005 e
                  ON e.CODIGO_ESTAGIO = l.CODIGO_ESTAGIO
                WHERE l.PERIODO_PRODUCAO = %s
                  AND l.ORDEM_CONFECCAO = %s
                  AND l.QTDE_EM_PRODUCAO_PACOTE > 0
            '''
            cursor.execute(sql, [periodo, ordem_confeccao])
            data = rows_to_dict_list(cursor)
            if len(data) == 0:
                context = {
                    'lote': lote,
                    'codigo_estagio': 'X',
                    'descricao_estagio': 'LOTE NÃO ENCONTRADO',
                }
            else:
                row = data[0]
                context = {
                    'lote': lote,
                    'codigo_estagio': row['CODIGO_ESTAGIO'],
                    'descricao_estagio': row['DESCRICAO_ESTAGIO'],
                }
    else:
        form = LoteForm()
    context['form'] = form
    return render(request, 'lotes/posicao.html', context)

def get_op(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.ORDEM_PRODUCAO OP
        , l.NOME_PROGRAMA_CRIACAO || ' - ' || p.DESCRICAO PRG
        , l.SITUACAO_ORDEM SITU
        , o.DATA_HORA DT
        FROM PCPC_040 l
        JOIN HDOC_036 p
          ON p.CODIGO_PROGRAMA = l.NOME_PROGRAMA_CRIACAO
         AND p.LOCALE = 'pt_BR'
        LEFT JOIN PCPC_020 o
          ON o.ORDEM_PRODUCAO = l.ORDEM_PRODUCAO
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
          AND rownum = 1
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    situacoes = {
        2: 'ORDEM CONF. GERADA',
        4: 'ORDENS EM PRODUCAO',
        9: 'ORDEM CANCELADA',
    }
    for row in data:
        row['SITU'] = '{} - {}'.format(row['SITU'], situacoes[row['SITU']])
    context.update({
        'o_headers': ('OP', 'Situação', 'Programa', 'Data/hora'),
        'o_fields': ('OP', 'SITU', 'PRG', 'DT'),
        'o_data': data,
    })
    return True

def get_item(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.PROCONF_NIVEL99
          || '.' || l.PROCONF_GRUPO
          || '.' || l.PROCONF_SUBGRUPO
          || '.' || l.PROCONF_ITEM ITEM
        , i.NARRATIVA NARR
        , l.QTDE_PROGRAMADA QTDE
        FROM PCPC_040 l
        JOIN BASI_010 i
          ON i.NIVEL_ESTRUTURA = l.PROCONF_NIVEL99
         AND i.GRUPO_ESTRUTURA = l.PROCONF_GRUPO
         AND i.SUBGRU_ESTRUTURA = l.PROCONF_SUBGRUPO
         AND i.ITEM_ESTRUTURA = l.PROCONF_ITEM
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
          AND rownum = 1
        ORDER BY
          l.SEQ_OPERACAO
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    context.update({
        'i_headers': ('Item', 'Descrição', 'Quantidade'),
        'i_fields': ('ITEM', 'NARR', 'QTDE'),
        'i_data': data,
    })
    return True

def get_estagios(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.CODIGO_ESTAGIO || ' - ' || e.DESCRICAO EST
        , l.QTDE_PROGRAMADA Q_P
        , l.QTDE_EM_PRODUCAO_PACOTE Q_EP
        , l.QTDE_A_PRODUZIR_PACOTE Q_AP
        , l.CODIGO_FAMILIA FAMI
        , l.NUMERO_ORDEM OS
        , coalesce(d.USUARIO_SYSTEXTIL, ' ') USU
        , d.DATA_INSERCAO DT
        , coalesce(d.PROCESSO_SYSTEXTIL || ' - ' || p.DESCRICAO, ' ') PRG
        FROM PCPC_040 l
        JOIN MQOP_005 e
          ON e.CODIGO_ESTAGIO = l.CODIGO_ESTAGIO
        LEFT JOIN PCPC_045 d
          ON d.PCPC040_PERCONF = l.PERIODO_PRODUCAO
         AND d.PCPC040_ORDCONF = l.ORDEM_CONFECCAO
         AND d.PCPC040_ESTCONF = l.CODIGO_ESTAGIO
        LEFT JOIN HDOC_036 p
          ON p.CODIGO_PROGRAMA = d.PROCESSO_SYSTEXTIL
         AND p.LOCALE = 'pt_BR'
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
        ORDER BY
          l.SEQ_OPERACAO
        , d.SEQUENCIA
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    for row in data:
        if row['DT'] is None:
            row['DT'] = ''
    context.update({
        'e_headers': ('Estágio', 'Prog.', 'Em Prod.',
                      'A Prod.', 'Família', 'Usuário', 'Data',
                      'Programa', 'OS'),
        'e_fields': ('EST', 'Q_P', 'Q_EP',
                     'Q_AP', 'FAMI', 'USU', 'DT', 'PRG', 'OS'),
        'e_data': data,
    })
    return True

def detalhes_lote(request, lote):
    periodo = lote[:4]
    ordem_confeccao = lote[-5:]
    cursor = connections['so'].cursor()

    context = {
        'l_headers': ('Período', 'OC'),
        'l_fields': ('PER', 'OC'),
        'l_data': [{
            'PER': periodo,
            'OC': ordem_confeccao,
        }],
    }

    if not get_op(cursor, context, periodo, ordem_confeccao):
        return HttpResponse('')

    get_item(cursor, context, periodo, ordem_confeccao)

    get_estagios(cursor, context, periodo, ordem_confeccao)

    html = render_to_string('lotes/ajax/detalhes_lote.html', context)
    return HttpResponse(html)
<commit_msg>dates<commit_after>from django.shortcuts import render
from django.db import connections
from django.http import HttpResponse
from django.template.loader import render_to_string

from .forms import LoteForm

def rows_to_dict_list(cursor):
    columns = [i[0] for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def index(request):
    context = {}
    return render(request, 'lotes/index.html', context)

def posicao(request):
    context = {}
    if request.method == 'POST':
        form = LoteForm(request.POST)
        if form.is_valid():
            lote = form.cleaned_data['lote']
            periodo = lote[:4]
            ordem_confeccao = lote[-5:]
            cursor = connections['so'].cursor()
            sql = '''
                SELECT
                  l.CODIGO_ESTAGIO
                , e.DESCRICAO DESCRICAO_ESTAGIO
                , l.QTDE_EM_PRODUCAO_PACOTE
                FROM PCPC_040 l
                JOIN MQOP_005 e
                  ON e.CODIGO_ESTAGIO = l.CODIGO_ESTAGIO
                WHERE l.PERIODO_PRODUCAO = %s
                  AND l.ORDEM_CONFECCAO = %s
                  AND l.QTDE_EM_PRODUCAO_PACOTE > 0
            '''
            cursor.execute(sql, [periodo, ordem_confeccao])
            data = rows_to_dict_list(cursor)
            if len(data) == 0:
                context = {
                    'lote': lote,
                    'codigo_estagio': 'X',
                    'descricao_estagio': 'LOTE NÃO ENCONTRADO',
                }
            else:
                row = data[0]
                context = {
                    'lote': lote,
                    'codigo_estagio': row['CODIGO_ESTAGIO'],
                    'descricao_estagio': row['DESCRICAO_ESTAGIO'],
                }
    else:
        form = LoteForm()
    context['form'] = form
    return render(request, 'lotes/posicao.html', context)

def get_periodo_oc(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          p.PERIODO_PRODUCAO PERIODO
        , TO_CHAR(p.DATA_INI_PERIODO, 'DD/MM/YYYY') INI
        , TO_CHAR(p.DATA_FIM_PERIODO - 1, 'DD/MM/YYYY') FIM
        , %s OC
        FROM PCPC_010 p
        WHERE AREA_PERIODO = 1
          AND PERIODO_PRODUCAO = %s
    '''
    cursor.execute(sql, [ordem_confeccao, periodo])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    context.update({
        'l_headers': ('Período', 'Incício', 'Fim', 'OC'),
        'l_fields': ('PERIODO', 'INI', 'FIM', 'OC'),
        'l_data': data,
    })
    return True

def get_op(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.ORDEM_PRODUCAO OP
        , l.NOME_PROGRAMA_CRIACAO || ' - ' || p.DESCRICAO PRG
        , l.SITUACAO_ORDEM SITU
        , TO_CHAR(o.DATA_HORA, 'DD/MM/YYYY HH24:MI') DT
        FROM PCPC_040 l
        JOIN HDOC_036 p
          ON p.CODIGO_PROGRAMA = l.NOME_PROGRAMA_CRIACAO
         AND p.LOCALE = 'pt_BR'
        LEFT JOIN PCPC_020 o
          ON o.ORDEM_PRODUCAO = l.ORDEM_PRODUCAO
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
          AND rownum = 1
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    situacoes = {
        2: 'ORDEM CONF. GERADA',
        4: 'ORDENS EM PRODUCAO',
        9: 'ORDEM CANCELADA',
    }
    for row in data:
        row['SITU'] = '{} - {}'.format(row['SITU'], situacoes[row['SITU']])
    context.update({
        'o_headers': ('OP', 'Situação', 'Programa', 'Data/hora'),
        'o_fields': ('OP', 'SITU', 'PRG', 'DT'),
        'o_data': data,
    })
    return True

def get_item(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.PROCONF_NIVEL99
          || '.' || l.PROCONF_GRUPO
          || '.' || l.PROCONF_SUBGRUPO
          || '.' || l.PROCONF_ITEM ITEM
        , i.NARRATIVA NARR
        , l.QTDE_PROGRAMADA QTDE
        FROM PCPC_040 l
        JOIN BASI_010 i
          ON i.NIVEL_ESTRUTURA = l.PROCONF_NIVEL99
         AND i.GRUPO_ESTRUTURA = l.PROCONF_GRUPO
         AND i.SUBGRU_ESTRUTURA = l.PROCONF_SUBGRUPO
         AND i.ITEM_ESTRUTURA = l.PROCONF_ITEM
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
          AND rownum = 1
        ORDER BY
          l.SEQ_OPERACAO
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    context.update({
        'i_headers': ('Item', 'Descrição', 'Quantidade'),
        'i_fields': ('ITEM', 'NARR', 'QTDE'),
        'i_data': data,
    })
    return True

def get_estagios(cursor, context, periodo, ordem_confeccao):
    sql = '''
        SELECT
          l.CODIGO_ESTAGIO || ' - ' || e.DESCRICAO EST
        , l.QTDE_PROGRAMADA Q_P
        , l.QTDE_EM_PRODUCAO_PACOTE Q_EP
        , l.QTDE_A_PRODUZIR_PACOTE Q_AP
        , l.CODIGO_FAMILIA FAMI
        , l.NUMERO_ORDEM OS
        , coalesce(d.USUARIO_SYSTEXTIL, ' ') USU
        , TO_CHAR(d.DATA_INSERCAO, 'DD/MM/YYYY HH24:MI') DT
        , coalesce(d.PROCESSO_SYSTEXTIL || ' - ' || p.DESCRICAO, ' ') PRG
        FROM PCPC_040 l
        JOIN MQOP_005 e
          ON e.CODIGO_ESTAGIO = l.CODIGO_ESTAGIO
        LEFT JOIN PCPC_045 d
          ON d.PCPC040_PERCONF = l.PERIODO_PRODUCAO
         AND d.PCPC040_ORDCONF = l.ORDEM_CONFECCAO
         AND d.PCPC040_ESTCONF = l.CODIGO_ESTAGIO
        LEFT JOIN HDOC_036 p
          ON p.CODIGO_PROGRAMA = d.PROCESSO_SYSTEXTIL
         AND p.LOCALE = 'pt_BR'
        WHERE l.PERIODO_PRODUCAO = %s
          AND l.ORDEM_CONFECCAO = %s
        ORDER BY
          l.SEQ_OPERACAO
        , d.SEQUENCIA
    '''
    cursor.execute(sql, [periodo, ordem_confeccao])
    data = rows_to_dict_list(cursor)
    if len(data) == 0:
        return False
    for row in data:
        if row['DT'] is None:
            row['DT'] = ''
    context.update({
        'e_headers': ('Estágio', 'Prog.', 'Em Prod.',
                      'A Prod.', 'Família', 'Usuário', 'Data',
                      'Programa', 'OS'),
        'e_fields': ('EST', 'Q_P', 'Q_EP',
                     'Q_AP', 'FAMI', 'USU', 'DT', 'PRG', 'OS'),
        'e_data': data,
    })
    return True

def detalhes_lote(request, lote):
    periodo = lote[:4]
    ordem_confeccao = lote[-5:]
    cursor = connections['so'].cursor()
    context = {}

    if not get_periodo_oc(cursor, context, periodo, ordem_confeccao):
        return HttpResponse('')

    if not get_op(cursor, context, periodo, ordem_confeccao):
        return HttpResponse('')

    get_item(cursor, context, periodo, ordem_confeccao)

    get_estagios(cursor, context, periodo, ordem_confeccao)

    html = render_to_string('lotes/ajax/detalhes_lote.html', context)
    return HttpResponse(html)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import division, print_function

import sys
import numpy as np

__author__ = "Ignacio Casso, Daniel Gamo, Gwydion J. Martín, Alberto Terceño"

class MLP(object):

    # Here are some appreciations about notation and the structure of vectors
    # and matrix:
    # N = data number
    # R = layers number (without the imput layer, as it has no activation
    #     functions nor weights subindex will be used to name these layers,
    #     with 0 being the input layer.
    # Dk = number of neurons on layer k

    # The weights' matrix W for each layer will have dimension (Dk, Dk+1)
    # Wij is the i-th weight of the j-th neuron on layer k+1. This decision is
    # forced because of the template, so:
    # to operate with a units vector on the matrix, you have to multiply by the
    # left and so both the activations and the units will be row vectors.

    # The matrix that group vectors with N different data (like the matrix x or
    # y) will have a row for each data, so they'll have dimension (N,?).

    # The lists of weights' and biases' matrix have the k-th layer data in the
    # (k-1)-th index. It's important to keep this phase shift in mind

    # self.nb_layers = R

    def __init__(self, K_list,
                 activation_functions, diff_activation_functions,
                 init_seed=None):

        self.K_list = K_list
        self.nb_layers = len(K_list) - 1  # = R

        # We suppose they're lists of R elements
        self.activation_functions = activation_functions
        # and that the k-th index represents the (k+1)-th layer
        self.diff_activation_functions = diff_activation_functions

        self.init_seed = init_seed

        self.weights_list = None  # list of R (Dk,Dk+1) matrix
        self.biases_list = None  # list of R row vectors of Dk+1 elements
        self.grad_w_list = None  # list of R (Dk,Dk+1) matrix
        self.grad_b_list = None  # list of R row vectors of Dk+1 elements

        self.activations = None  # list of R+1 (N,Dk) matrix
        self.units = None  # list of R+1 (N,Dk) matrix
        self.y = None  # (N,Dr) matrix

        self.init_weights()

# %% definition of activation functions and derivatives
    @staticmethod
    def sigmoid(z):
        return 1 / (1 + np.exp(-z))

    @staticmethod
    def dsigmoid(z):
        return MLP.sigmoid(z) * (1 - MLP.sigmoid(z))

    @staticmethod
    def dtanh(z):
        return 1 - np.tanh(z)**2

    @staticmethod
    def relu(z):
        return np.maximum(z, 0)

    @staticmethod
    def drelu(z):
        z[z >= 0] = 1   # drelu(0)=1 by agreement
        z[z < 0] = 0
        return z

    @staticmethod
    def identity(z):
        return z

    @staticmethod
    def didentity(z):  # it only works with numpy arrays
        return [1] * z.shape[0]

    @staticmethod
    def softmax(z):
        sum_exp = np.sum(np.exp(z))
        return np.exp(z) / sum_exp

    # %% cost functions
    @staticmethod
    def binary_cross_entropy(y, t_data):
        return -np.sum(t_data * np.log(y) + (1 - t_data) * np.log(1 - y))

    @staticmethod
    def softmax_cross_entropy(y, t_data):
        return -np.sum(t_data * np.log(y))

    @staticmethod
    def cost_L2(y, t_data):
        return 0.5*np.sum((y - t_data)**2)

    # %% simple weights initialization

    def init_weights(self):

        if self.init_seed:
            np.random.seed(self.seed)

        weights_list = []
        biases_list = []

        for layer in range(self.nb_layers):
            new_W = np.random.randn(self.K_list[layer], self.K_list[layer + 1])
            new_b = np.zeros(self.K_list[layer + 1])
            weights_list.append(new_W)
            biases_list.append(new_b)

        self.weights_list = weights_list
        self.biases_list = biases_list

    # %% feed forward pass
    # x = (N,D0) matrix
    def get_activations_and_units(self, x):

        activations = [x]
        units = [x]
        z = x
        for i in range(self.nb_layers):
            # matrix + row vector, so it adds the vector to each of the matrix
            # rows
            a = z.dot(self.weights_list[i]) + self.biases_list[i]
            activations.append(a)
            z = self.activation_functions[i](a)
            units.append(z)

        self.activations = activations
        self.units = units
        self.y = z

    # %% backpropagation
    # This function calculates the error gradient for each of the data and
    # averages them. All the gradients are calculated at the same time using
    # (N,?) matrix instead of vectors.
    # We use : x = (N,D0) matrix, t = (N,Dr) matrix, delta_k = (N,Dk) matrix
    def get_gradients(self, x, t, beta=0):

        # Slightly different from the class notes due to the separation of bs
        # and Ws and the change of the index to name the weights.
        # The functions returns a list of shifted index (k-th index = (k+1)-th
        # layer gradients; the layer 0 (input) has no Ws)

        self.get_activations_and_units(x)

        N = x.shape[0]
        grad_w_list = [0]*self.nb_layers
        grad_b_list = [0]*self.nb_layers

        delta_k1 = None  # delta value for the next layer

        ks = range(1, self.nb_layers+1)
        ks.reverse()
        for k in ks:  # r, ..., 1

            # we calculate the new delta values
            if (k < self.nb_layers):
                # weights of the (k+1)-th layer
                w = self.weights_list[k]
                # activation function derivative on layer k
                dh = self.diff_activation_functions[k-1]
                # activations from layer k
                a = self.activations[k]
                delta_k = (delta_k1.dot(w.T))*dh(a)
            else:
                # we can assume the derivative of En respect to the last
                # activations layer is y-t
                delta_k = self.y - t

            grad_wk = (np.einsum('ij,ik', self.units[k-1], delta_k)/N) + (beta * self.weights_list[k-1])
            grad_w_list[k-1] = grad_wk

            grad_bk = np.sum(delta_k, axis=0)/N + beta*self.biases_list[k-1]
            grad_b_list[k-1] = grad_bk

            delta_k1 = delta_k

        ##

        self.grad_w_list = grad_w_list
        self.grad_b_list = grad_b_list

    # %%
    # training method for the neuron
    def train(self, x_data, t_data,
              epochs, batch_size,
              initialize_weights=False,
              epsilon=0.01,
              beta=0,
              print_cost=False):

        if initialize_weights:
            self.init_weights()

        nb_data = x_data.shape[0]
        index_list = np.arange(nb_data)
        nb_batches = int(nb_data / batch_size)

        for _ in range(epochs):
            np.random.shuffle(index_list)
            for batch in range(nb_batches):
                indexes = index_list[batch*batch_size:(batch+1)*batch_size]
                self.get_gradients(x_data[indexes], t_data[indexes], beta)
                self.weights_list = [(self.weights_list[k] -
                                     epsilon*self.grad_w_list[k])
                                     for k in range(self.nb_layers)]
                self.biases_list = [(self.biases_list[k] -
                                    epsilon*self.grad_b_list[k])
                                    for k in range(self.nb_layers)]

            if print_cost:
                x_batch = x_data
                t_batch = t_data
                self.get_activations_and_units(x_batch)
                if self.activation_functions[-1] == MLP.sigmoid:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.binary_cross_entropy(self.y, t_batch))
                    sys.stdout.flush()
                elif self.activation_functions[-1] == MLP.softmax:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.softmax_cross_entropy(
                                         self.y, t_batch))
                    sys.stdout.flush()
                else:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.cost_L2(self.y, t_batch))
                    sys.stdout.flush()

# %% let's experiment

if __name__ == '__main__':

    # %% Create data
    # np.random.seed(5)
    nb_black = 50
    nb_red = 50
    nb_data = nb_black + nb_red
    x_data_black = np.random.randn(nb_black, 2) + np.array([0, 0])
    x_data_red = np.random.randn(nb_red, 2) + np.array([10, 10])

    x_data = np.vstack((x_data_black, x_data_red))
    t_data = np.asarray([0]*nb_black + [1]*nb_red).reshape(nb_data, 1)

# %% Net structure
    D = x_data.shape[1]  # initial dimension
    K = 1  # final dimension

    K_list = [D, K]  # list of dimensions

    activation_functions = [MLP.sigmoid]
    diff_activation_functions = [MLP.dsigmoid]

# %%
    mlp = MLP(K_list, activation_functions, diff_activation_functions)

# %% Train begins
    mlp.train(x_data, t_data,
              epochs=1000, batch_size=10, initialize_weights=True, epsilon=0.1,
              print_cost=True)
<commit_msg>Mlp actualizado<commit_after># -*- coding: utf-8 -*-

from __future__ import division, print_function

import sys
import numpy as np

__author__ = "Ignacio Casso, Daniel Gamo, Gwydion J. Martín, Alberto Terceño"

class MLP(object):

    # Here are some appreciations about notation and the structure of vectors
    # and matrix:
    # N = data number
    # R = layers number (without the imput layer, as it has no activation
    #     functions nor weights subindex will be used to name these layers,
    #     with 0 being the input layer.
    # Dk = number of neurons on layer k

    # The weights' matrix W for each layer will have dimension (Dk, Dk+1)
    # Wij is the i-th weight of the j-th neuron on layer k+1. This decision is
    # forced because of the template, so:
    # to operate with a units vector on the matrix, you have to multiply by the
    # left and so both the activations and the units will be row vectors.

    # The matrix that group vectors with N different data (like the matrix x or
    # y) will have a row for each data, so they'll have dimension (N,?).

    # The lists of weights' and biases' matrix have the k-th layer data in the
    # (k-1)-th index. It's important to keep this phase shift in mind

    # self.nb_layers = R

    def __init__(self, K_list,
                 activation_functions, diff_activation_functions,
                 init_seed=None):

        self.K_list = K_list
        self.nb_layers = len(K_list) - 1  # = R

        # We suppose they're lists of R elements
        self.activation_functions = activation_functions
        # and that the k-th index represents the (k+1)-th layer
        self.diff_activation_functions = diff_activation_functions

        self.init_seed = init_seed

        self.weights_list = None  # list of R (Dk,Dk+1) matrix
        self.biases_list = None  # list of R row vectors of Dk+1 elements
        self.grad_w_list = None  # list of R (Dk,Dk+1) matrix
        self.grad_b_list = None  # list of R row vectors of Dk+1 elements

        self.activations = None  # list of R+1 (N,Dk) matrix
        self.units = None  # list of R+1 (N,Dk) matrix
        self.y = None  # (N,Dr) matrix

        # TODO
        self.dict_method = {'SGD':sgd, 'Momentum':momentum, 'Nesterov':nesterov}
        self.dict_param = {'SGD':[], 'Momentum':[], 'Nesterov':{'v_b':[0]*self.nb_layers, 'v_w':[0]*self.nb_layers }}

        self.init_weights()

# %% definition of activation functions and derivatives
    @staticmethod
    def sigmoid(z):
        return 1 / (1 + np.exp(-z))

    @staticmethod
    def dsigmoid(z):
        return MLP.sigmoid(z) * (1 - MLP.sigmoid(z))

    @staticmethod
    def dtanh(z):
        return 1 - np.tanh(z)**2

    @staticmethod
    def relu(z):
        return np.maximum(z, 0)

    @staticmethod
    def drelu(z):
        z[z >= 0] = 1   # drelu(0)=1 by agreement
        z[z < 0] = 0
        return z

    @staticmethod
    def identity(z):
        return z

    @staticmethod
    def didentity(z):  # it only works with numpy arrays
        return [1] * z.shape[0]

    @staticmethod
    def softmax(z):
        sum_exp = np.sum(np.exp(z))
        return np.exp(z) / sum_exp

    # %% cost functions
    @staticmethod
    def binary_cross_entropy(y, t_data):
        return -np.sum(t_data * np.log(y) + (1 - t_data) * np.log(1 - y))

    @staticmethod
    def softmax_cross_entropy(y, t_data):
        return -np.sum(t_data * np.log(y))

    @staticmethod
    def cost_L2(y, t_data):
        return 0.5*np.sum((y - t_data)**2)

    # %% simple weights initialization

    def init_weights(self):

        if self.init_seed:
            np.random.seed(self.seed)

        weights_list = []
        biases_list = []

        for layer in range(self.nb_layers):
            new_W = np.random.randn(self.K_list[layer], self.K_list[layer + 1])
            new_b = np.zeros(self.K_list[layer + 1])
            weights_list.append(new_W)
            biases_list.append(new_b)

        self.weights_list = weights_list
        self.biases_list = biases_list

    # %% feed forward pass
    # x = (N,D0) matrix
    def get_activations_and_units(self, x):

        activations = [x]
        units = [x]
        z = x
        for i in range(self.nb_layers):
            # matrix + row vector, so it adds the vector to each of the matrix
            # rows
            a = z.dot(self.weights_list[i]) + self.biases_list[i]
            activations.append(a)
            z = self.activation_functions[i](a)
            units.append(z)

        self.activations = activations
        self.units = units
        self.y = z

    # %% backpropagation
    # This function calculates the error gradient for each of the data and
    # averages them. All the gradients are calculated at the same time using
    # (N,?) matrix instead of vectors.
    # We use : x = (N,D0) matrix, t = (N,Dr) matrix, delta_k = (N,Dk) matrix
    def get_gradients(self, x, t, beta=0):

        # Slightly different from the class notes due to the separation of bs
        # and Ws and the change of the index to name the weights.
        # The functions returns a list of shifted index (k-th index = (k+1)-th
        # layer gradients; the layer 0 (input) has no Ws)

        self.get_activations_and_units(x)

        N = x.shape[0]
        grad_w_list = [0]*self.nb_layers
        grad_b_list = [0]*self.nb_layers

        delta_k1 = None  # delta value for the next layer

        ks = range(1, self.nb_layers+1)
        ks.reverse()
        for k in ks:  # r, ..., 1

            # we calculate the new delta values
            if (k < self.nb_layers):
                # weights of the (k+1)-th layer
                w = self.weights_list[k]
                # activation function derivative on layer k
                dh = self.diff_activation_functions[k-1]
                # activations from layer k
                a = self.activations[k]
                delta_k = (delta_k1.dot(w.T))*dh(a)
            else:
                # we can assume the derivative of En respect to the last
                # activations layer is y-t
                delta_k = self.y - t

            grad_wk = (np.einsum('ij,ik', self.units[k-1], delta_k)/N) + (beta * self.weights_list[k-1])
            grad_w_list[k-1] = grad_wk

            grad_bk = np.sum(delta_k, axis=0)/N
            grad_b_list[k-1] = grad_bk

            delta_k1 = delta_k

        ##

        self.grad_w_list = grad_w_list
        self.grad_b_list = grad_b_list

    # %%
    # training method for the neuron
    def train(self, x_data, t_data,
              epochs, batch_size,
              initialize_weights=False,
              method='SGD',
              epsilon=0.01,
              beta=0,
              gamma=0.9,
              print_cost=False):

        def sgd():
            self.get_gradients(x_data[indexes], t_data[indexes], beta)
            self.weights_list = [(self.weights_list[k] -
                                     epsilon*self.grad_w_list[k])
                                     for k in range(self.nb_layers)]
            self.biases_list = [(self.biases_list[k] -
                                    epsilon*self.grad_b_list[k])
                                    for k in range(self.nb_layers)]

        def momentum():

        def nesterov():
            # TODO
            aux_w = [gamma*self.dict_param['Nesterov'][k] for k in range(self.nb_layers)]
            aux_b =
            self.weights_list = [self.weights_list[k] - aux[k]
                                     for k in range(self.nb_layers)]
            self.biases_list = [self.biases_list[k] - aux
                                    for k in range(self.nb_layers)]
            self.get_gradients(x_data[indexes], t_data[indexes], beta)
            self.dict_param['Nesterov'] = [gamma*self.dict_param['Nesterov'][k] + epsilon*self.weights_list[k] for k in range(self.nb_layers)]

        def adagrad():

        def adam():

        if initialize_weights:
            self.init_weights()

        nb_data = x_data.shape[0]
        index_list = np.arange(nb_data)
        nb_batches = int(nb_data / batch_size)

        for _ in range(epochs):
            np.random.shuffle(index_list)
            for batch in range(nb_batches):
                indexes = index_list[batch*batch_size:(batch+1)*batch_size]
                dict_method[method]()

            if print_cost:
                x_batch = x_data
                t_batch = t_data
                self.get_activations_and_units(x_batch)
                if self.activation_functions[-1] == MLP.sigmoid:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.binary_cross_entropy(self.y, t_batch))
                    sys.stdout.flush()
                elif self.activation_functions[-1] == MLP.softmax:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.softmax_cross_entropy(
                                         self.y, t_batch))
                    sys.stdout.flush()
                else:
                    sys.stdout.write('cost = %f\r' %
                                     MLP.cost_L2(self.y, t_batch))
                    sys.stdout.flush()

# %% let's experiment

if __name__ == '__main__':

    # %% Create data
    # np.random.seed(5)
    nb_black = 50
    nb_red = 50
    nb_data = nb_black + nb_red
    x_data_black = np.random.randn(nb_black, 2) + np.array([0, 0])
    x_data_red = np.random.randn(nb_red, 2) + np.array([10, 10])

    x_data = np.vstack((x_data_black, x_data_red))
    t_data = np.asarray([0]*nb_black + [1]*nb_red).reshape(nb_data, 1)

# %% Net structure
    D = x_data.shape[1]  # initial dimension
    K = 1  # final dimension

    K_list = [D, K]  # list of dimensions

    activation_functions = [MLP.sigmoid]
    diff_activation_functions = [MLP.dsigmoid]

# %%
    mlp = MLP(K_list, activation_functions, diff_activation_functions)

# %% Train begins
    mlp.train(x_data, t_data,
              epochs=1000, batch_size=10, initialize_weights=True, epsilon=0.1,
              print_cost=True)
<|endoftext|>


<commit_before>import os
import sys
import json
import logging
import data_helper
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.contrib import learn

logging.getLogger().setLevel(logging.INFO)

def predict_unseen_data():
	"""Step 0: load trained model and parameters"""
	params = json.loads(open('./parameters.json').read())
	checkpoint_dir = sys.argv[1]
	if not checkpoint_dir.endswith('/'):
		checkpoint_dir += '/'
	checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints')
	logging.critical('Loaded the trained model: {}'.format(checkpoint_file))

	"""Step 1: load data for prediction"""
	test_file = sys.argv[2]
	test_examples = json.loads(open(test_file).read())

	# labels.json was saved during training, and it has to be loaded during prediction
	labels = json.loads(open('./labels.json').read())
	one_hot = np.zeros((len(labels), len(labels)), int)
	np.fill_diagonal(one_hot, 1)
	label_dict = dict(zip(labels, one_hot))

	x_raw = [example['consumer_complaint_narrative'] for example in test_examples]
	x_test = [data_helper.clean_str(x) for x in x_raw]
	logging.info('The number of x_test: {}'.format(len(x_test)))

	y_test = None
	if 'product' in test_examples[0]:
		y_raw = [example['product'] for example in test_examples]
		y_test = [label_dict[y] for y in y_raw]
		logging.info('The number of y_test: {}'.format(len(y_test)))

	vocab_path = os.path.join(checkpoint_dir, "vocab.pickle")
	vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
	x_test = np.array(list(vocab_processor.transform(x_raw)))

	"""Step 2: compute the predictions"""
	graph = tf.Graph()
	with graph.as_default():
		session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
		sess = tf.Session(config=session_conf)

		with sess.as_default():
			saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
			saver.restore(sess, checkpoint_file)

			input_x = graph.get_operation_by_name("input_x").outputs[0]
			dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
			predictions = graph.get_operation_by_name("output/predictions").outputs[0]

			batches = data_helper.batch_iter(list(x_test), params['batch_size'], 1, shuffle=False)
			all_predictions = []
			for x_test_batch in batches:
				batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
				all_predictions = np.concatenate([all_predictions, batch_predictions])

	if y_test is not None:
		y_test = np.argmax(y_test, axis=1)
		correct_predictions = sum(all_predictions == y_test)
		logging.critical('The accuracy is: {}'.format(correct_predictions / float(len(y_test))))

if __name__ == '__main__':
	predict_unseen_data()
<commit_msg>predict with system arguments<commit_after>import os
import sys
import json
import logging
import data_helper
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.contrib import learn

logging.getLogger().setLevel(logging.INFO)

def predict_unseen_data():
	"""Step 0: load trained model and parameters"""
	params = json.loads(open('./parameters.json').read())
	checkpoint_dir = sys.argv[1]
	if not checkpoint_dir.endswith('/'):
		checkpoint_dir += '/'
	checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints')
	logging.critical('Loaded the trained model: {}'.format(checkpoint_file))

	"""Step 1: load data for prediction"""
	test_file = sys.argv[2]
	test_examples = json.loads(open(test_file).read())

	# labels.json was saved during training, and it has to be loaded during prediction
	labels = json.loads(open('./labels.json').read())
	one_hot = np.zeros((len(labels), len(labels)), int)
	np.fill_diagonal(one_hot, 1)
	label_dict = dict(zip(labels, one_hot))

	x_raw = [example['consumer_complaint_narrative'] for example in test_examples]
	x_test = [data_helper.clean_str(x) for x in x_raw]
	logging.info('The number of x_test: {}'.format(len(x_test)))

	y_test = None
	if 'product' in test_examples[0]:
		y_raw = [example['product'] for example in test_examples]
		y_test = [label_dict[y] for y in y_raw]
		logging.info('The number of y_test: {}'.format(len(y_test)))

	vocab_path = os.path.join(checkpoint_dir, "vocab.pickle")
	vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
	x_test = np.array(list(vocab_processor.transform(x_raw)))

	"""Step 2: compute the predictions"""
	graph = tf.Graph()
	with graph.as_default():
		session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
		sess = tf.Session(config=session_conf)

		with sess.as_default():
			saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
			saver.restore(sess, checkpoint_file)

			input_x = graph.get_operation_by_name("input_x").outputs[0]
			dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
			predictions = graph.get_operation_by_name("output/predictions").outputs[0]

			batches = data_helper.batch_iter(list(x_test), params['batch_size'], 1, shuffle=False)
			all_predictions = []
			for x_test_batch in batches:
				batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
				all_predictions = np.concatenate([all_predictions, batch_predictions])

	if y_test is not None:
		y_test = np.argmax(y_test, axis=1)
		correct_predictions = sum(all_predictions == y_test)
		logging.critical('The accuracy is: {}'.format(correct_predictions / float(len(y_test))))

if __name__ == '__main__':
	# python3 predict.py ./trained_model_1478649295/ ./data/small_samples.json
	predict_unseen_data()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import argparse
import errno
import os
import re
import readline
import select
import signal
import socket
import sys
import threading
import time

try:
    input = raw_input
except NameError:
    pass

SIG_NAMES = {}
SIG_NUMBERS = set()
for sig, num in vars(signal).items():
    if sig.startswith('SIG') and '_' not in sig:
        SIG_NAMES[sig] = num
        SIG_NAMES[sig[3:]] = num
        SIG_NUMBERS.add(num)

def parse_pid(value, regex=re.compile(r'^.+(/manhole-)?(?P<pid>\d+)$')):
    match = regex.match(value)
    if not match:
        raise argparse.ArgumentTypeError("PID must be in one of these forms: 1234 or /tmp/manhole-1234")

    return int(match.group('pid'))

def parse_signal(value):
    try:
        value = int(value)
    except ValueError:
        pass
    else:
        if value in SIG_NUMBERS:
            return value
        else:
            raise argparse.ArgumentTypeError("Invalid signal number %s. Expected one of: %s" % (
                value, ', '.join(str(i) for i in SIG_NUMBERS)
            ))
    value = value.upper()
    if value in SIG_NAMES:
        return SIG_NAMES[value]
    else:
        raise argparse.ArgumentTypeError("Invalid signal name %r." % value)

parser = argparse.ArgumentParser(description='Connect to a manhole.')
parser.add_argument('pid', metavar='PID', type=parse_pid,  # nargs='?',
                    help='A numerical process id, or a path in the form: /tmp/manhole-1234')
parser.add_argument('-t', '--timeout', dest='timeout', default=1, type=float,
                    help='Timeout to use. Default: %(default)s seconds.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-1', '-USR1', dest='signal', action='store_const', const=int(signal.SIGUSR1),
                   help='Send USR1 (%(const)s) to the process before connecting.')
group.add_argument('-2', '-USR2', dest='signal', action='store_const', const=int(signal.SIGUSR2),
                   help='Send USR2 (%(const)s) to the process before connecting.')
group.add_argument('-s', '--signal', dest='signal', type=parse_signal, metavar="SIGNAL",
                   help='Send the given SIGNAL to the process before connecting.')

class ConnectionHandler(threading.Thread):
    def __init__(self, timeout, sock, read_fd=None, wait_the_end=True):
        super(ConnectionHandler, self).__init__()
        self.sock = sock
        self.read_fd = read_fd
        self.conn_fd = sock.fileno()
        self.timeout = timeout
        self.should_run = True
        self._poller = select.poll()
        self.wait_the_end = wait_the_end

    def run(self):
        if self.read_fd is not None:
            self._poller.register(self.read_fd, select.POLLIN)
        self._poller.register(self.conn_fd, select.POLLIN)

        while self.should_run:
            self.poll()
        if self.wait_the_end:
            t = time.time()
            while time.time() - t < self.timeout:
                self.poll()

    def poll(self):
        milliseconds = self.timeout * 1000
        for fd, _ in self._poller.poll(milliseconds):
            if fd == self.conn_fd:
                data = self.sock.recv(1024*1024)
                sys.stdout.write(data.decode('utf8'))
                sys.stdout.flush()
                readline.redisplay()
            elif fd == self.read_fd:
                data = os.read(self.read_fd, 1024)
                self.sock.sendall(data)
            else:
                raise RuntimeError("Unknown FD %s" % fd)

def main():
    args = parser.parse_args()

    histfile = os.path.join(os.path.expanduser("~"), ".manhole_history")
    try:
        readline.read_history_file(histfile)
    except IOError:
        pass
    import atexit

    atexit.register(readline.write_history_file, histfile)
    del histfile

    if args.signal:
        os.kill(args.pid, args.signal)

    start = time.time()
    uds_path = '/tmp/manhole-%s' % args.pid
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.settimeout(args.timeout)
    while time.time() - start < args.timeout:
        try:
            sock.connect(uds_path)
        except Exception as exc:
            if exc.errno not in (errno.ENOENT, errno.ECONNREFUSED):
                print("Failed to connect to %r: %r" % (uds_path, exc), file=sys.stderr)
        else:
            break
    else:
        print("Failed to connect to %r: Timeout" % uds_path, file=sys.stderr)
        sys.exit(5)

    read_fd, write_fd = os.pipe()

    thread = ConnectionHandler(args.timeout, sock, read_fd, not sys.stdin.isatty())
    thread.start()

    try:
        while thread.is_alive():
            try:
                data = input()
            except EOFError:
                break
            os.write(write_fd, data.encode('utf8'))
            os.write(write_fd, b'\n')
    except KeyboardInterrupt:
        pass
    finally:
        thread.should_run = False
        thread.join()
<commit_msg>Fix buggy regex (make the prefix optional).<commit_after>#!/usr/bin/env python
from __future__ import print_function

import argparse
import errno
import os
import re
import readline
import select
import signal
import socket
import sys
import threading
import time

try:
    input = raw_input
except NameError:
    pass

SIG_NAMES = {}
SIG_NUMBERS = set()
for sig, num in vars(signal).items():
    if sig.startswith('SIG') and '_' not in sig:
        SIG_NAMES[sig] = num
        SIG_NAMES[sig[3:]] = num
        SIG_NUMBERS.add(num)

def parse_pid(value, regex=re.compile(r'^(.*/manhole-)?(?P<pid>\d+)$')):
    match = regex.match(value)
    if not match:
        raise argparse.ArgumentTypeError("PID must be in one of these forms: 1234 or /tmp/manhole-1234")

    return int(match.group('pid'))

def parse_signal(value):
    try:
        value = int(value)
    except ValueError:
        pass
    else:
        if value in SIG_NUMBERS:
            return value
        else:
            raise argparse.ArgumentTypeError("Invalid signal number %s. Expected one of: %s" % (
                value, ', '.join(str(i) for i in SIG_NUMBERS)
            ))
    value = value.upper()
    if value in SIG_NAMES:
        return SIG_NAMES[value]
    else:
        raise argparse.ArgumentTypeError("Invalid signal name %r." % value)

parser = argparse.ArgumentParser(description='Connect to a manhole.')
parser.add_argument('pid', metavar='PID', type=parse_pid,  # nargs='?',
                    help='A numerical process id, or a path in the form: /tmp/manhole-1234')
parser.add_argument('-t', '--timeout', dest='timeout', default=1, type=float,
                    help='Timeout to use. Default: %(default)s seconds.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-1', '-USR1', dest='signal', action='store_const', const=int(signal.SIGUSR1),
                   help='Send USR1 (%(const)s) to the process before connecting.')
group.add_argument('-2', '-USR2', dest='signal', action='store_const', const=int(signal.SIGUSR2),
                   help='Send USR2 (%(const)s) to the process before connecting.')
group.add_argument('-s', '--signal', dest='signal', type=parse_signal, metavar="SIGNAL",
                   help='Send the given SIGNAL to the process before connecting.')

class ConnectionHandler(threading.Thread):
    def __init__(self, timeout, sock, read_fd=None, wait_the_end=True):
        super(ConnectionHandler, self).__init__()
        self.sock = sock
        self.read_fd = read_fd
        self.conn_fd = sock.fileno()
        self.timeout = timeout
        self.should_run = True
        self._poller = select.poll()
        self.wait_the_end = wait_the_end

    def run(self):
        if self.read_fd is not None:
            self._poller.register(self.read_fd, select.POLLIN)
        self._poller.register(self.conn_fd, select.POLLIN)

        while self.should_run:
            self.poll()
        if self.wait_the_end:
            t = time.time()
            while time.time() - t < self.timeout:
                self.poll()

    def poll(self):
        milliseconds = self.timeout * 1000
        for fd, _ in self._poller.poll(milliseconds):
            if fd == self.conn_fd:
                data = self.sock.recv(1024*1024)
                sys.stdout.write(data.decode('utf8'))
                sys.stdout.flush()
                readline.redisplay()
            elif fd == self.read_fd:
                data = os.read(self.read_fd, 1024)
                self.sock.sendall(data)
            else:
                raise RuntimeError("Unknown FD %s" % fd)

def main():
    args = parser.parse_args()

    histfile = os.path.join(os.path.expanduser("~"), ".manhole_history")
    try:
        readline.read_history_file(histfile)
    except IOError:
        pass
    import atexit

    atexit.register(readline.write_history_file, histfile)
    del histfile

    if args.signal:
        os.kill(args.pid, args.signal)

    start = time.time()
    uds_path = '/tmp/manhole-%s' % args.pid
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.settimeout(args.timeout)
    while time.time() - start < args.timeout:
        try:
            sock.connect(uds_path)
        except Exception as exc:
            if exc.errno not in (errno.ENOENT, errno.ECONNREFUSED):
                print("Failed to connect to %r: %r" % (uds_path, exc), file=sys.stderr)
        else:
            break
    else:
        print("Failed to connect to %r: Timeout" % uds_path, file=sys.stderr)
        sys.exit(5)

    read_fd, write_fd = os.pipe()

    thread = ConnectionHandler(args.timeout, sock, read_fd, not sys.stdin.isatty())
    thread.start()

    try:
        while thread.is_alive():
            try:
                data = input()
            except EOFError:
                break
            os.write(write_fd, data.encode('utf8'))
            os.write(write_fd, b'\n')
    except KeyboardInterrupt:
        pass
    finally:
        thread.should_run = False
        thread.join()
<|endoftext|>


<commit_before>'''
This is the default app controller for portality.
For inclusion in your own project you should make your own version of this controller
and include the views you require, as well as writing new ones. Of course, views must
also be backed up by models, so have a look at the example models and use them / write
new ones as required too.
'''
import os

from flask import Flask, request, abort, render_template, redirect, send_file
from flask.views import View
from flask.ext.login import login_user, current_user

import portality.models as models
from portality.core import app, login_manager
from portality import settings

from portality.view.account import blueprint as account
from portality.view.admin import blueprint as admin
from portality.view.publisher import blueprint as publisher
from portality.view.query import blueprint as query
from portality.view.stream import blueprint as stream
from portality.view.forms import blueprint as forms
from portality.view.doaj import blueprint as doaj
from portality.view.oaipmh import blueprint as oaipmh
from portality.view.atom import blueprint as atom

app.register_blueprint(account, url_prefix='/account')
app.register_blueprint(admin, url_prefix='/admin')
app.register_blueprint(publisher, url_prefix='/publisher')
app.register_blueprint(query, url_prefix='/query')
app.register_blueprint(query, url_prefix="/admin_query")
app.register_blueprint(query, url_prefix="/publisher_query")
app.register_blueprint(stream, url_prefix='/stream')
app.register_blueprint(forms, url_prefix='/forms')
app.register_blueprint(oaipmh)
app.register_blueprint(atom)
app.register_blueprint(doaj)

"""
FIXME: this needs to be sorted out - they shouldn't be in here and in doaj.py, but there is an issue
with the 404 pages which requires them
"""
import sys
try:
    if sys.version_info.major == 2 and sys.version_info.minor < 7:
        from portality.ordereddict import OrderedDict
except AttributeError:
    if sys.version_info[0] == 2 and sys.version_info[1] < 7:
        from portality.ordereddict import OrderedDict
    else:
        from collections import OrderedDict
else:
    from collections import OrderedDict
SPONSORS = {
        # the key should correspond to the sponsor logo name in
        # /static/doaj/images/sponsors without the extension for
        # consistency - no code should rely on this though
        'biomed-central': {'name':'BioMed Central', 'logo': 'biomed-central.gif', 'url': 'http://www.biomedcentral.com/'},
        'coaction': {'name': 'Co-Action Publishing', 'logo': 'coaction.gif', 'url': 'http://www.co-action.net/'},
        'cogent-oa': {'name': 'Cogent OA', 'logo': 'cogent-oa.gif', 'url': 'http://cogentoa.com/'},
        'copernicus': {'name': 'Copernicus Publications', 'logo': 'copernicus.gif', 'url': 'http://publications.copernicus.org/'},
        'dovepress': {'name': 'Dove Medical Press', 'logo': 'dovepress.png', 'url': 'http://www.dovepress.com/'},
        'frontiers': {'name': 'Frontiers', 'logo': 'frontiers.gif', 'url': 'http://www.frontiersin.org/'},
        'hindawi': {'name': 'Hindawi Publishing Corporation', 'logo': 'hindawi.jpg', 'url': 'http://www.hindawi.com/'},
        'inasp': {'name': 'International Network for the Availability of Scientific Publications (INASP)', 'logo': 'inasp.png', 'url': 'http://www.inasp.info/'},
        'lund-university': {'name': 'Lund University', 'logo': 'lund-university.jpg', 'url': 'http://www.lunduniversity.lu.se/'},
        'mdpi': {'name': 'Multidisciplinary Digital Publishing Institute (MDPI)', 'logo': 'mdpi.png', 'url': 'http://www.mdpi.com/'},
        'springer': {'name': 'Springer Science+Business Media', 'logo': 'springer.gif', 'url': 'http://www.springer.com/'},
        'taylor-and-francis': {'name': 'Taylor and Francis Group', 'logo': 'taylor-and-francis.gif', 'url': 'http://www.taylorandfrancisgroup.com/'},
}
SPONSORS = OrderedDict(sorted(SPONSORS.items(), key=lambda t: t[0])) # create an ordered dictionary, sort by the key of the unordered one

# Redirects from previous DOAJ app.
# RJ: I have decided to put these here so that they can be managed
# alongside the DOAJ codebase.  I know they could also go into the
# nginx config, but there is a chance that they will get lost or forgotten
# some day, whereas this approach doesn't have that risk.
@app.route("/doaj")
def legacy():
    func = request.values.get("func")
    if func == "csv":
        return redirect("/csv"), 301
    elif func == "rss":
        return redirect("/feed"), 301
    abort(404)

@app.route("/doaj2csv")
def another_legacy_csv_route():
    return redirect("/csv"), 301

@app.route("/schemas/doajArticles.xsd")
def legacy_doaj_XML_schema():
    return send_file(os.path.join(app.config.get("STATIC_DIR"), "doaj", 'doajArticles.xsd'), mimetype="application/xml")

@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = models.Account.pull(userid)
    return out

# FIXME: this used to calculate the site stats on request, but for the time being
# this is an unnecessary overhead, so taking it out.  Will need to put something
# equivalent back in when we do the admin area
@app.context_processor
def set_current_context():
    """ Set some template context globals. """
    '''
    Inserts variables into every template this blueprint renders.  This
    one deals with the announcement in the header, which can't be built
    into the template directly, as various styles are applied only if a
    header is present on the page. It also makes the list of DOAJ
    sponsors available and may include similar minor pieces of
    information.
    '''
    return {
        'heading_title': '',
        'heading_text': '',
        'sponsors': SPONSORS,
        'settings': settings,
        'statistics' : models.JournalArticle.site_statistics(),
        "current_user": current_user,
        "app" : app
        }
    # return dict(current_user=current_user, app=app)

@app.before_request
def standard_authentication():
    """Check remote_user on a per-request basis."""
    remote_user = request.headers.get('REMOTE_USER', '')
    if remote_user:
        user = models.Account.pull(remote_user)
        if user:
            login_user(user, remember=False)
    # add a check for provision of api key
    elif 'api_key' in request.values:
        res = models.Account.query(q='api_key:"' + request.values['api_key'] + '"')['hits']['hits']
        if len(res) == 1:
            user = models.Account.pull(res[0]['_source']['id'])
            if user:
                login_user(user, remember=False)

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.errorhandler(401)
def page_not_found(e):
    return render_template('401.html'), 401

if __name__ == "__main__":
    app.run(host='0.0.0.0', debug=app.config['DEBUG'], port=app.config['PORT'])

<commit_msg>send DOAJ XML schema file as attachment, mimic nginx behaviour on the new route<commit_after>'''
This is the default app controller for portality.
For inclusion in your own project you should make your own version of this controller
and include the views you require, as well as writing new ones. Of course, views must
also be backed up by models, so have a look at the example models and use them / write
new ones as required too.
'''
import os

from flask import Flask, request, abort, render_template, redirect, send_file
from flask.views import View
from flask.ext.login import login_user, current_user

import portality.models as models
from portality.core import app, login_manager
from portality import settings

from portality.view.account import blueprint as account
from portality.view.admin import blueprint as admin
from portality.view.publisher import blueprint as publisher
from portality.view.query import blueprint as query
from portality.view.stream import blueprint as stream
from portality.view.forms import blueprint as forms
from portality.view.doaj import blueprint as doaj
from portality.view.oaipmh import blueprint as oaipmh
from portality.view.atom import blueprint as atom

app.register_blueprint(account, url_prefix='/account')
app.register_blueprint(admin, url_prefix='/admin')
app.register_blueprint(publisher, url_prefix='/publisher')
app.register_blueprint(query, url_prefix='/query')
app.register_blueprint(query, url_prefix="/admin_query")
app.register_blueprint(query, url_prefix="/publisher_query")
app.register_blueprint(stream, url_prefix='/stream')
app.register_blueprint(forms, url_prefix='/forms')
app.register_blueprint(oaipmh)
app.register_blueprint(atom)
app.register_blueprint(doaj)

"""
FIXME: this needs to be sorted out - they shouldn't be in here and in doaj.py, but there is an issue
with the 404 pages which requires them
"""
import sys
try:
    if sys.version_info.major == 2 and sys.version_info.minor < 7:
        from portality.ordereddict import OrderedDict
except AttributeError:
    if sys.version_info[0] == 2 and sys.version_info[1] < 7:
        from portality.ordereddict import OrderedDict
    else:
        from collections import OrderedDict
else:
    from collections import OrderedDict
SPONSORS = {
        # the key should correspond to the sponsor logo name in
        # /static/doaj/images/sponsors without the extension for
        # consistency - no code should rely on this though
        'biomed-central': {'name':'BioMed Central', 'logo': 'biomed-central.gif', 'url': 'http://www.biomedcentral.com/'},
        'coaction': {'name': 'Co-Action Publishing', 'logo': 'coaction.gif', 'url': 'http://www.co-action.net/'},
        'cogent-oa': {'name': 'Cogent OA', 'logo': 'cogent-oa.gif', 'url': 'http://cogentoa.com/'},
        'copernicus': {'name': 'Copernicus Publications', 'logo': 'copernicus.gif', 'url': 'http://publications.copernicus.org/'},
        'dovepress': {'name': 'Dove Medical Press', 'logo': 'dovepress.png', 'url': 'http://www.dovepress.com/'},
        'frontiers': {'name': 'Frontiers', 'logo': 'frontiers.gif', 'url': 'http://www.frontiersin.org/'},
        'hindawi': {'name': 'Hindawi Publishing Corporation', 'logo': 'hindawi.jpg', 'url': 'http://www.hindawi.com/'},
        'inasp': {'name': 'International Network for the Availability of Scientific Publications (INASP)', 'logo': 'inasp.png', 'url': 'http://www.inasp.info/'},
        'lund-university': {'name': 'Lund University', 'logo': 'lund-university.jpg', 'url': 'http://www.lunduniversity.lu.se/'},
        'mdpi': {'name': 'Multidisciplinary Digital Publishing Institute (MDPI)', 'logo': 'mdpi.png', 'url': 'http://www.mdpi.com/'},
        'springer': {'name': 'Springer Science+Business Media', 'logo': 'springer.gif', 'url': 'http://www.springer.com/'},
        'taylor-and-francis': {'name': 'Taylor and Francis Group', 'logo': 'taylor-and-francis.gif', 'url': 'http://www.taylorandfrancisgroup.com/'},
}
SPONSORS = OrderedDict(sorted(SPONSORS.items(), key=lambda t: t[0])) # create an ordered dictionary, sort by the key of the unordered one

# Redirects from previous DOAJ app.
# RJ: I have decided to put these here so that they can be managed
# alongside the DOAJ codebase.  I know they could also go into the
# nginx config, but there is a chance that they will get lost or forgotten
# some day, whereas this approach doesn't have that risk.
@app.route("/doaj")
def legacy():
    func = request.values.get("func")
    if func == "csv":
        return redirect("/csv"), 301
    elif func == "rss":
        return redirect("/feed"), 301
    abort(404)

@app.route("/doaj2csv")
def another_legacy_csv_route():
    return redirect("/csv"), 301

@app.route("/schemas/doajArticles.xsd")
def legacy_doaj_XML_schema():
    schema_fn = 'doajArticles.xsd'
    return send_file(
            os.path.join(app.config.get("STATIC_DIR"), "doaj", schema_fn),
            mimetype="application/xml", as_attachment=True, attachment_filename=schema_fn
            )

@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = models.Account.pull(userid)
    return out

# FIXME: this used to calculate the site stats on request, but for the time being
# this is an unnecessary overhead, so taking it out.  Will need to put something
# equivalent back in when we do the admin area
@app.context_processor
def set_current_context():
    """ Set some template context globals. """
    '''
    Inserts variables into every template this blueprint renders.  This
    one deals with the announcement in the header, which can't be built
    into the template directly, as various styles are applied only if a
    header is present on the page. It also makes the list of DOAJ
    sponsors available and may include similar minor pieces of
    information.
    '''
    return {
        'heading_title': '',
        'heading_text': '',
        'sponsors': SPONSORS,
        'settings': settings,
        'statistics' : models.JournalArticle.site_statistics(),
        "current_user": current_user,
        "app" : app
        }
    # return dict(current_user=current_user, app=app)

@app.before_request
def standard_authentication():
    """Check remote_user on a per-request basis."""
    remote_user = request.headers.get('REMOTE_USER', '')
    if remote_user:
        user = models.Account.pull(remote_user)
        if user:
            login_user(user, remember=False)
    # add a check for provision of api key
    elif 'api_key' in request.values:
        res = models.Account.query(q='api_key:"' + request.values['api_key'] + '"')['hits']['hits']
        if len(res) == 1:
            user = models.Account.pull(res[0]['_source']['id'])
            if user:
                login_user(user, remember=False)

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.errorhandler(401)
def page_not_found(e):
    return render_template('401.html'), 401

if __name__ == "__main__":
    app.run(host='0.0.0.0', debug=app.config['DEBUG'], port=app.config['PORT'])

<|endoftext|>