<commit_before>import json
import settings
import ynab_client

from flask import Flask, request, jsonify, redirect
from datetime import datetime
from dateutil.parser import parse
from decimal import Decimal

from pynYNAB.schema.budget import Transaction

app = Flask('Fintech to YNAB')
app.config['DEBUG'] = settings.flask_debug

@app.route('/')
def route_index():
    return redirect("https://github.com/scottrobertson/fintech-to-ynab", code=302)

@app.route('/ping')
def route_ping():
    return 'pong';

@app.route('/starling', methods=['POST'])
def route_starling():
    if settings.url_secret is not None and settings.url_secret != request.args.get('secret'):
        return jsonify({'error': 'Invalid secret'}), 403

    ynab_client.init()
    data = json.loads(request.data.decode('utf8'))
    settings.log.debug('webhook type received %s', data['content']['type'])
    if data['content']['type'] in ['TRANSACTION_CARD', 'TRANSACTION_FASTER_PAYMENT_IN', 'TRANSACTION_FASTER_PAYMENT_OUT', 'TRANSACTION_DIRECT_DEBIT']:
        body, code = create_transaction_from_starling(data, settings, 0)
        return jsonify(body), code
    else:
        settings.log.warning('Unsupported webhook type: %s', data['content']['type'])
        return jsonify({'error': 'Unsupported webhook type'}), 400

@app.route('/monzo', methods=['POST'])
def route_monzo():
    if settings.url_secret is not None and settings.url_secret != request.args.get('secret'):
        return jsonify({'error': 'Invalid secret'}), 403

    ynab_client.init()
    data = json.loads(request.data.decode('utf8'))
    settings.log.debug('webhook type received %s', data['type'])
    if data.get('type') == 'transaction.created':
        body, code = create_transaction_from_monzo(data['data'], settings, 0)
        return jsonify(body), code
    else:
        settings.log.warning('Unsupported webhook type: %s', data['type'])
        return jsonify({'error': 'Unsupported webhook type'}), 400

def create_transaction_from_starling(data, settings, expected_delta):
    # Sync the account so we get the latest payees
    ynab_client.sync()

    if data['content']['amount'] == 0:
        return {'error': 'Transaction amount is 0.'}, 200

    # Does this account exist?
    account = ynab_client.getaccount(settings.starling_ynab_account)
    if not account:
        return {'error': 'Account {} was not found'.format(settings.starling_ynab_account)}, 400

    payee_name = data['content']['counterParty']
    subcategory_id = None
    flag = None
    cleared = None
    memo = ''

    # If we are creating the payee, then we need to increase the delta
    if ynab_client.payeeexists(payee_name):
        settings.log.debug('payee exists, using %s', payee_name)
        subcategory_id = get_subcategory_from_payee(payee_name)
    else:
        settings.log.debug('payee does not exist, will create %s', payee_name)
        expected_delta += 1

    entities_payee_id = ynab_client.getpayee(payee_name).id

    if data['content']['sourceCurrency'] != 'GBP':
        memo += ' (%s %s)' % (data['content']['sourceCurrency'], abs(data['content']['sourceAmount']))
        flag = 'Orange'
    else:
        cleared = 'Cleared'

    # Create the Transaction
    expected_delta += 1
    settings.log.debug('Creating transaction object')
    transaction = Transaction(
        check_number=data['content'].get('transactionUid'),
        entities_account_id=account.id,
        amount=data['content']['amount'],
        date=parse(data['timestamp']),
        entities_payee_id=entities_payee_id,
        imported_date=datetime.now().date(),
        imported_payee=payee_name,
        source="Imported",
        flag=flag,
        cleared=cleared,
        memo=memo
    )

    if subcategory_id is not None:
        transaction.entities_subcategory_id = subcategory_id

    settings.log.debug('Duplicate detection')
    if ynab_client.containsDuplicate(transaction):
        settings.log.debug('skipping due to duplicate transaction')
        return {'error': 'Tried to add a duplicate transaction.'}, 200
    else:
        settings.log.debug('appending and pushing transaction to YNAB. Delta: %s', expected_delta)
        ynab_client.client.budget.be_transactions.append(transaction)
        ynab_client.client.push(expected_delta)
        return {'message': 'Transaction created in YNAB successfully.'}, 201

def create_transaction_from_monzo(data, settings, expected_delta):
    # Sync the account so we get the latest payees
    ynab_client.sync()

    if data['amount'] == 0:
        return {'error': 'Transaction amount is 0.'}, 200

    # Does this account exist?
    account = ynab_client.getaccount(settings.monzo_ynab_account)
    if not account:
        return {'error': 'Account {} was not found'.format(settings.monzo_ynab_account)}, 400

    # Work out the Payee Name
    if data.get('merchant'):
        payee_name = data['merchant']['name']
        subcategory_id = get_subcategory_from_payee(payee_name)
    else:
        # This is a p2p transaction
        payee_name = get_p2p_transaction_payee_name(data)
        subcategory_id = None

    # If we are creating the payee, then we need to increase the delta
    if not ynab_client.payeeexists(payee_name):
        settings.log.debug('payee does not exist, will create %s', payee_name)
        expected_delta += 1

    # Get the payee ID. This will append a new one if needed
    entities_payee_id = ynab_client.getpayee(payee_name).id

    memo = ''
    if settings.include_emoji and data['merchant'] and data['merchant'].get('emoji'):
        memo += ' %s' % data['merchant']['emoji']

    if settings.include_tags and data['merchant'] and data['merchant'].get('metadata', {}).get('suggested_tags'):
        memo += ' %s' % data['merchant']['metadata']['suggested_tags']

    # Show the local currency in the notes if this is not in the accounts currency
    flag = None
    cleared = None
    if data['local_currency'] != data['currency']:
        memo += ' (%s %s)' % (data['local_currency'], (abs(data['local_amount']) / 100))
        flag = 'Orange'
    else:
        cleared = 'Cleared'

    # Create the Transaction
    expected_delta += 1
    settings.log.debug('Creating transaction object')
    transaction = Transaction(
        check_number=data['id'],
        entities_account_id=account.id,
        amount=Decimal(data['amount']) / 100,
        date=parse(data['created']),
        entities_payee_id=entities_payee_id,
        imported_date=datetime.now().date(),
        imported_payee=payee_name,
        memo=memo,
        source="Imported",
        flag=flag,
        cleared=cleared
    )

    if subcategory_id is not None:
        transaction.entities_subcategory_id = subcategory_id

    settings.log.debug('Duplicate detection')
    if ynab_client.containsDuplicate(transaction):
        settings.log.debug('skipping due to duplicate transaction')
        return {'error': 'Tried to add a duplicate transaction.'}, 200
    else:
        settings.log.debug('appending and pushing transaction to YNAB. Delta: %s', expected_delta)
        ynab_client.client.budget.be_transactions.append(transaction)
        ynab_client.client.push(expected_delta)
        return {'message': 'Transaction created in YNAB successfully.'}, 201

def get_subcategory_from_payee(payee_name):
    """
    Get payee details for a previous transaction in YNAB.
    If a payee with payee_name has been used in the past, we can get their ID and
    pre-populate category.

    :param payee_name: The name of the Payee as coming from the bank.
    :return: (payee_id, subcategory_id)
    """
    previous_transaction = ynab_client.findPreviousTransaction(payee_name)
    if previous_transaction is not None and previous_transaction.entities_payee is not None:
        settings.log.debug('A previous transaction for the payee %s has been found', payee_name)
        return get_subcategory_id_for_transaction(previous_transaction, payee_name)
    else:
        settings.log.debug('A previous transaction for the payee %s has not been found', payee_name)
    return None

def get_subcategory_id_for_transaction(transaction, payee_name):
    """
    Gets the subcategory ID for a transaction.
    Filters out transactions that have multiple categories.

    :param transaction: The transaction to get subcategory ID from.
    :return: The subcategory ID, or None if it is a multiple-category transaction.
    """
    subcategory = transaction.entities_subcategory

    if subcategory is not None:
        if subcategory.name != 'Split (Multiple Categories)...':
            settings.log.debug('We have identified the "%s" category as a good default for this payee', subcategory.name)
            return subcategory.id
        else:
            settings.log.debug('Split category found, so we will not use that category for %s', payee_name)
    else:
        settings.log.debug('A subcategory was not found for the previous transaction for %s', payee_name)

def get_p2p_transaction_payee_name(data):
    """
    Get the payee name for a p2p transaction, based on webook transaction data.

    :param data: The 'data' key of the transaction data.
    :return: The string name of the payee.
    """
    if data.get('counterparty'):
        if data['counterparty'].has_key('name'):
            payee_name = data['counterparty']['name']
        else:
            payee_name = data['counterparty']['number']
    elif data.get('metadata', {}).get('is_topup') == 'true':
        payee_name = 'Topup'
    else:
        payee_name = 'Unknown Payee'

    return payee_name

if __name__ == "__main__":
    ynab_client.init()
    ynab_client.sync()
    app.run(host='0.0.0.0', port=settings.port)
<commit_msg>Starr logging full requests<commit_after>import json
import settings
import ynab_client

from flask import Flask, request, jsonify, redirect
from datetime import datetime
from dateutil.parser import parse
from decimal import Decimal

from pynYNAB.schema.budget import Transaction

app = Flask('Fintech to YNAB')
app.config['DEBUG'] = settings.flask_debug

@app.route('/')
def route_index():
    return redirect("https://github.com/scottrobertson/fintech-to-ynab", code=302)

@app.route('/ping')
def route_ping():
    return 'pong';

@app.route('/starling', methods=['POST'])
def route_starling():
    if settings.url_secret is not None and settings.url_secret != request.args.get('secret'):
        return jsonify({'error': 'Invalid secret'}), 403

    settings.log.debug(request.data)

    ynab_client.init()
    data = json.loads(request.data.decode('utf8'))
    settings.log.debug('webhook type received %s', data['content']['type'])
    if data['content']['type'] in ['TRANSACTION_CARD', 'TRANSACTION_FASTER_PAYMENT_IN', 'TRANSACTION_FASTER_PAYMENT_OUT', 'TRANSACTION_DIRECT_DEBIT']:
        body, code = create_transaction_from_starling(data, settings, 0)
        return jsonify(body), code
    else:
        settings.log.warning('Unsupported webhook type: %s', data['content']['type'])
        return jsonify({'error': 'Unsupported webhook type'}), 400

@app.route('/monzo', methods=['POST'])
def route_monzo():
    if settings.url_secret is not None and settings.url_secret != request.args.get('secret'):
        return jsonify({'error': 'Invalid secret'}), 403

    settings.log.debug(request.data)

    ynab_client.init()
    data = json.loads(request.data.decode('utf8'))
    settings.log.debug('webhook type received %s', data['type'])
    if data.get('type') == 'transaction.created':
        body, code = create_transaction_from_monzo(data['data'], settings, 0)
        return jsonify(body), code
    else:
        settings.log.warning('Unsupported webhook type: %s', data['type'])
        return jsonify({'error': 'Unsupported webhook type'}), 400

def create_transaction_from_starling(data, settings, expected_delta):
    # Sync the account so we get the latest payees
    ynab_client.sync()

    if data['content']['amount'] == 0:
        return {'error': 'Transaction amount is 0.'}, 200

    # Does this account exist?
    account = ynab_client.getaccount(settings.starling_ynab_account)
    if not account:
        return {'error': 'Account {} was not found'.format(settings.starling_ynab_account)}, 400

    payee_name = data['content']['counterParty']
    subcategory_id = None
    flag = None
    cleared = None
    memo = ''

    # If we are creating the payee, then we need to increase the delta
    if ynab_client.payeeexists(payee_name):
        settings.log.debug('payee exists, using %s', payee_name)
        subcategory_id = get_subcategory_from_payee(payee_name)
    else:
        settings.log.debug('payee does not exist, will create %s', payee_name)
        expected_delta += 1

    entities_payee_id = ynab_client.getpayee(payee_name).id

    if data['content']['sourceCurrency'] != 'GBP':
        memo += ' (%s %s)' % (data['content']['sourceCurrency'], abs(data['content']['sourceAmount']))
        flag = 'Orange'
    else:
        cleared = 'Cleared'

    # Create the Transaction
    expected_delta += 1
    settings.log.debug('Creating transaction object')
    transaction = Transaction(
        check_number=data['content'].get('transactionUid'),
        entities_account_id=account.id,
        amount=data['content']['amount'],
        date=parse(data['timestamp']),
        entities_payee_id=entities_payee_id,
        imported_date=datetime.now().date(),
        imported_payee=payee_name,
        source="Imported",
        flag=flag,
        cleared=cleared,
        memo=memo
    )

    if subcategory_id is not None:
        transaction.entities_subcategory_id = subcategory_id

    settings.log.debug('Duplicate detection')
    if ynab_client.containsDuplicate(transaction):
        settings.log.debug('skipping due to duplicate transaction')
        return {'error': 'Tried to add a duplicate transaction.'}, 200
    else:
        settings.log.debug('appending and pushing transaction to YNAB. Delta: %s', expected_delta)
        ynab_client.client.budget.be_transactions.append(transaction)
        ynab_client.client.push(expected_delta)
        return {'message': 'Transaction created in YNAB successfully.'}, 201

def create_transaction_from_monzo(data, settings, expected_delta):
    # Sync the account so we get the latest payees
    ynab_client.sync()

    if data['amount'] == 0:
        return {'error': 'Transaction amount is 0.'}, 200

    # Does this account exist?
    account = ynab_client.getaccount(settings.monzo_ynab_account)
    if not account:
        return {'error': 'Account {} was not found'.format(settings.monzo_ynab_account)}, 400

    # Work out the Payee Name
    if data.get('merchant'):
        payee_name = data['merchant']['name']
        subcategory_id = get_subcategory_from_payee(payee_name)
    else:
        # This is a p2p transaction
        payee_name = get_p2p_transaction_payee_name(data)
        subcategory_id = None

    # If we are creating the payee, then we need to increase the delta
    if not ynab_client.payeeexists(payee_name):
        settings.log.debug('payee does not exist, will create %s', payee_name)
        expected_delta += 1

    # Get the payee ID. This will append a new one if needed
    entities_payee_id = ynab_client.getpayee(payee_name).id

    memo = ''
    if settings.include_emoji and data['merchant'] and data['merchant'].get('emoji'):
        memo += ' %s' % data['merchant']['emoji']

    if settings.include_tags and data['merchant'] and data['merchant'].get('metadata', {}).get('suggested_tags'):
        memo += ' %s' % data['merchant']['metadata']['suggested_tags']

    # Show the local currency in the notes if this is not in the accounts currency
    flag = None
    cleared = None
    if data['local_currency'] != data['currency']:
        memo += ' (%s %s)' % (data['local_currency'], (abs(data['local_amount']) / 100))
        flag = 'Orange'
    else:
        cleared = 'Cleared'

    # Create the Transaction
    expected_delta += 1
    settings.log.debug('Creating transaction object')
    transaction = Transaction(
        check_number=data['id'],
        entities_account_id=account.id,
        amount=Decimal(data['amount']) / 100,
        date=parse(data['created']),
        entities_payee_id=entities_payee_id,
        imported_date=datetime.now().date(),
        imported_payee=payee_name,
        memo=memo,
        source="Imported",
        flag=flag,
        cleared=cleared
    )

    if subcategory_id is not None:
        transaction.entities_subcategory_id = subcategory_id

    settings.log.debug('Duplicate detection')
    if ynab_client.containsDuplicate(transaction):
        settings.log.debug('skipping due to duplicate transaction')
        return {'error': 'Tried to add a duplicate transaction.'}, 200
    else:
        settings.log.debug('appending and pushing transaction to YNAB. Delta: %s', expected_delta)
        ynab_client.client.budget.be_transactions.append(transaction)
        ynab_client.client.push(expected_delta)
        return {'message': 'Transaction created in YNAB successfully.'}, 201

def get_subcategory_from_payee(payee_name):
    """
    Get payee details for a previous transaction in YNAB.
    If a payee with payee_name has been used in the past, we can get their ID and
    pre-populate category.

    :param payee_name: The name of the Payee as coming from the bank.
    :return: (payee_id, subcategory_id)
    """
    previous_transaction = ynab_client.findPreviousTransaction(payee_name)
    if previous_transaction is not None and previous_transaction.entities_payee is not None:
        settings.log.debug('A previous transaction for the payee %s has been found', payee_name)
        return get_subcategory_id_for_transaction(previous_transaction, payee_name)
    else:
        settings.log.debug('A previous transaction for the payee %s has not been found', payee_name)
    return None

def get_subcategory_id_for_transaction(transaction, payee_name):
    """
    Gets the subcategory ID for a transaction.
    Filters out transactions that have multiple categories.

    :param transaction: The transaction to get subcategory ID from.
    :return: The subcategory ID, or None if it is a multiple-category transaction.
    """
    subcategory = transaction.entities_subcategory

    if subcategory is not None:
        if subcategory.name != 'Split (Multiple Categories)...':
            settings.log.debug('We have identified the "%s" category as a good default for this payee', subcategory.name)
            return subcategory.id
        else:
            settings.log.debug('Split category found, so we will not use that category for %s', payee_name)
    else:
        settings.log.debug('A subcategory was not found for the previous transaction for %s', payee_name)

def get_p2p_transaction_payee_name(data):
    """
    Get the payee name for a p2p transaction, based on webook transaction data.

    :param data: The 'data' key of the transaction data.
    :return: The string name of the payee.
    """
    if data.get('counterparty'):
        if data['counterparty'].has_key('name'):
            payee_name = data['counterparty']['name']
        else:
            payee_name = data['counterparty']['number']
    elif data.get('metadata', {}).get('is_topup') == 'true':
        payee_name = 'Topup'
    else:
        payee_name = 'Unknown Payee'

    return payee_name

if __name__ == "__main__":
    ynab_client.init()
    ynab_client.sync()
    app.run(host='0.0.0.0', port=settings.port)
<|endoftext|>


<commit_before>#
# Copyright 2014 Google Inc. All rights reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#

"""Performs requests to the Google Maps Directions API."""

from googlemaps import convert

def directions(client, origin, destination,
               mode=None, waypoints=None, alternatives=False, avoid=None,
               language=None, units=None, region=None, departure_time=None,
               arrival_time=None, optimize_waypoints=False, transit_mode=None,
               transit_routing_preference=None, traffic_model=None):
    """Get directions between an origin point and a destination point.

    :param origin: The address or latitude/longitude value from which you wish
        to calculate directions.
    :type origin: string, dict, list, or tuple

    :param destination: The address or latitude/longitude value from which
        you wish to calculate directions. You can use a place_id as destination
        using 'place_id' as a preffix in the passing parameter.
    :type destination: string, dict, list, or tuple

    :param mode: Specifies the mode of transport to use when calculating
        directions. One of "driving", "walking", "bicycling" or "transit"
    :type mode: string

    :param waypoints: Specifies an array of waypoints. Waypoints alter a
        route by routing it through the specified location(s).
    :type waypoints: a single location, or a list of locations, where a
        location is a string, dict, list, or tuple

    :param alternatives: If True, more than one route may be returned in the
        response.
    :type alternatives: bool

    :param avoid: Indicates that the calculated route(s) should avoid the
        indicated features.
    :type avoid: list or string

    :param language: The language in which to return results.
    :type language: string

    :param units: Specifies the unit system to use when displaying results.
        "metric" or "imperial"
    :type units: string

    :param region: The region code, specified as a ccTLD ("top-level domain"
        two-character value.
    :type region: string

    :param departure_time: Specifies the desired time of departure.
    :type departure_time: int or datetime.datetime

    :param arrival_time: Specifies the desired time of arrival for transit
        directions. Note: you can't specify both departure_time and
        arrival_time.
    :type arrival_time: int or datetime.datetime

    :param optimize_waypoints: Optimize the provided route by rearranging the
        waypoints in a more efficient order.
    :type optimize_waypoints: bool

    :param transit_mode: Specifies one or more preferred modes of transit.
        This parameter may only be specified for requests where the mode is
        transit. Valid values are "bus", "subway", "train", "tram", "rail".
        "rail" is equivalent to ["train", "tram", "subway"].
    :type transit_mode: string or list of strings

    :param transit_routing_preference: Specifies preferences for transit
        requests. Valid values are "less_walking" or "fewer_transfers"
    :type transit_routing_preference: string

    :param traffic_model: Specifies the predictive travel time model to use.
        Valid values are "best_guess" or "optimistic" or "pessimistic".
        The traffic_model parameter may only be specified for requests where
        the travel mode is driving, and where the request includes a
        departure_time.
    :type units: string

    :rtype: list of routes
    """

    params = {
        "origin": convert.latlng(origin),
        "destination": convert.latlng(destination)
    }

    if mode:
        # NOTE(broady): the mode parameter is not validated by the Maps API
        # server. Check here to prevent silent failures.
        if mode not in ["driving", "walking", "bicycling", "transit"]:
            raise ValueError("Invalid travel mode.")
        params["mode"] = mode

    if waypoints:
        waypoints = convert.location_list(waypoints)
        if optimize_waypoints:
            waypoints = "optimize:true|" + waypoints
        params["waypoints"] = waypoints

    if alternatives:
        params["alternatives"] = "true"

    if avoid:
        params["avoid"] = convert.join_list("|", avoid)

    if language:
        params["language"] = language

    if units:
        params["units"] = units

    if region:
        params["region"] = region

    if departure_time:
        params["departure_time"] = convert.time(departure_time)

    if arrival_time:
        params["arrival_time"] = convert.time(arrival_time)

    if departure_time and arrival_time:
        raise ValueError("Should not specify both departure_time and"
                         "arrival_time.")

    if transit_mode:
        params["transit_mode"] = convert.join_list("|", transit_mode)

    if transit_routing_preference:
        params["transit_routing_preference"] = transit_routing_preference

    if traffic_model:
        params["traffic_model"] = traffic_model

    return client._request("/maps/api/directions/json", params).get("routes", [])
<commit_msg>Update directions.py<commit_after>#
# Copyright 2014 Google Inc. All rights reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#

"""Performs requests to the Google Maps Directions API."""

from googlemaps import convert

def directions(client, origin, destination,
               mode=None, waypoints=None, alternatives=False, avoid=None,
               language=None, units=None, region=None, departure_time=None,
               arrival_time=None, optimize_waypoints=False, transit_mode=None,
               transit_routing_preference=None, traffic_model=None):
    """Get directions between an origin point and a destination point.

    :param origin: The address or latitude/longitude value from which you wish
        to calculate directions.
    :type origin: string, dict, list, or tuple

    :param destination: The address or latitude/longitude value from which
        you wish to calculate directions. You can use a place_id as destination
        by putting 'place_id:' as a preffix in the passing parameter.
    :type destination: string, dict, list, or tuple

    :param mode: Specifies the mode of transport to use when calculating
        directions. One of "driving", "walking", "bicycling" or "transit"
    :type mode: string

    :param waypoints: Specifies an array of waypoints. Waypoints alter a
        route by routing it through the specified location(s).
    :type waypoints: a single location, or a list of locations, where a
        location is a string, dict, list, or tuple

    :param alternatives: If True, more than one route may be returned in the
        response.
    :type alternatives: bool

    :param avoid: Indicates that the calculated route(s) should avoid the
        indicated features.
    :type avoid: list or string

    :param language: The language in which to return results.
    :type language: string

    :param units: Specifies the unit system to use when displaying results.
        "metric" or "imperial"
    :type units: string

    :param region: The region code, specified as a ccTLD ("top-level domain"
        two-character value.
    :type region: string

    :param departure_time: Specifies the desired time of departure.
    :type departure_time: int or datetime.datetime

    :param arrival_time: Specifies the desired time of arrival for transit
        directions. Note: you can't specify both departure_time and
        arrival_time.
    :type arrival_time: int or datetime.datetime

    :param optimize_waypoints: Optimize the provided route by rearranging the
        waypoints in a more efficient order.
    :type optimize_waypoints: bool

    :param transit_mode: Specifies one or more preferred modes of transit.
        This parameter may only be specified for requests where the mode is
        transit. Valid values are "bus", "subway", "train", "tram", "rail".
        "rail" is equivalent to ["train", "tram", "subway"].
    :type transit_mode: string or list of strings

    :param transit_routing_preference: Specifies preferences for transit
        requests. Valid values are "less_walking" or "fewer_transfers"
    :type transit_routing_preference: string

    :param traffic_model: Specifies the predictive travel time model to use.
        Valid values are "best_guess" or "optimistic" or "pessimistic".
        The traffic_model parameter may only be specified for requests where
        the travel mode is driving, and where the request includes a
        departure_time.
    :type units: string

    :rtype: list of routes
    """

    params = {
        "origin": convert.latlng(origin),
        "destination": convert.latlng(destination)
    }

    if mode:
        # NOTE(broady): the mode parameter is not validated by the Maps API
        # server. Check here to prevent silent failures.
        if mode not in ["driving", "walking", "bicycling", "transit"]:
            raise ValueError("Invalid travel mode.")
        params["mode"] = mode

    if waypoints:
        waypoints = convert.location_list(waypoints)
        if optimize_waypoints:
            waypoints = "optimize:true|" + waypoints
        params["waypoints"] = waypoints

    if alternatives:
        params["alternatives"] = "true"

    if avoid:
        params["avoid"] = convert.join_list("|", avoid)

    if language:
        params["language"] = language

    if units:
        params["units"] = units

    if region:
        params["region"] = region

    if departure_time:
        params["departure_time"] = convert.time(departure_time)

    if arrival_time:
        params["arrival_time"] = convert.time(arrival_time)

    if departure_time and arrival_time:
        raise ValueError("Should not specify both departure_time and"
                         "arrival_time.")

    if transit_mode:
        params["transit_mode"] = convert.join_list("|", transit_mode)

    if transit_routing_preference:
        params["transit_routing_preference"] = transit_routing_preference

    if traffic_model:
        params["traffic_model"] = traffic_model

    return client._request("/maps/api/directions/json", params).get("routes", [])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2010 Dirk Holtwick, holtwick.it
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.1b2"
<commit_msg>Bump version<commit_after># -*- coding: utf-8 -*-
# Copyright 2010 Dirk Holtwick, holtwick.it
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.1b3"
<|endoftext|>


<commit_before>from OpenGL.GL import *
from OpenGL.GLUT import *
from shader import BaseShaderProgram
from object import Object
import os
import sys
import numpy as np
from matrix_transform import *
from math import pi, radians
from Camera import Camera

WIDTH = 800
HEIGHT = 600
FPS = 50
ESC = b'\033'
SIZE_OF_FLOAT = 4

window = 0
program_id = 0

# uniform variables
uniform_loc = {}
uniforms = ['myTextureSampler',
            'MVP', 'M', 'V', 'NormalMatrix',
            'lightPos_worldspace',
            'material.ambient',
            'material.diffuse',
            'material.specular',
            'material.shininess',
            ]

# motion control
g_camera = Camera()

lightPos_worldspace = np.array([0, 0, 3])

#Objects
boxObj = None
flowerObj = None
g_object_list = []

def reshape(w, h):
    """TODO"""
    glViewport(0, 0, w, h)

def display():
    global lightPos_worldspace, g_camera
    global boxObj, flowerObj
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

    glUseProgram(program_id)
    # set transform matrix
    # matrix_scale = scale(g_scale_ratio)
    # ModelMatrix = rotate(matrix_scale.T, g_verticalAngle, np.array([1, 0, 0], 'f'))
    # ModelMatrix = rotate(ModelMatrix.T, g_horizontalAngle, np.array([0, 1, 0], 'f'))
    # matrix_translate = translate(g_translate_x, g_translate_y, 0.0)
    # ModelMatrix = np.dot(matrix_translate, ModelMatrix)

    ProjectionMatrix = perspective(radians(g_camera.zoom), 4 / 3, 0.1, 100)
    ViewMatrix = g_camera.getViewMatrix()
    glUniformMatrix4fv(uniform_loc['V'], 1, GL_TRUE, c_matrix(ViewMatrix))

    # set light parameter
    lightPos = g_camera.getLightPosition()
    glUniform3f(uniform_loc['lightPos_worldspace'], lightPos[0], lightPos[1], lightPos[2])
    glUniform1f(uniform_loc['material.shininess'], 32)

    # display
    obj = boxObj
    ModelMatrix = np.identity(4, 'f')
    matrix_translate = translate(2, 0, -5)
    ModelMatrix = dots(matrix_translate, ModelMatrix)
    MVP = dots(ProjectionMatrix, ViewMatrix, ModelMatrix)
    glUniformMatrix4fv(uniform_loc['M'], 1, GL_TRUE, c_matrix(ModelMatrix))
    glUniformMatrix4fv(uniform_loc['MVP'], 1, GL_TRUE, c_matrix(MVP))
    for i in range(len(obj.geom_nums)):
        glBindVertexArray(obj.vao_ids[i])
        # mtl parameter
        mtl = obj.mtl_buffer_list[i]
        glUniform3f(uniform_loc['material.ambient'], mtl.Ka[0], mtl.Ka[1], mtl.Ka[2])
        glUniform3f(uniform_loc['material.diffuse'], mtl.Kd[0], mtl.Kd[1], mtl.Kd[2])
        glUniform3f(uniform_loc['material.specular'], mtl.Ks[0], mtl.Ks[1], mtl.Ks[2])

        # texture
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, obj.texture_ids[i])
        glUniform1i(uniform_loc['myTextureSampler'], 0)

        # vertex
        glBindBuffer(GL_ARRAY_BUFFER, obj.vertex_buffer_ids[i])
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, None)
        # uv
        glBindBuffer(GL_ARRAY_BUFFER, obj.uv_buffer_ids[i])
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, None)
        # normal
        glBindBuffer(GL_ARRAY_BUFFER, obj.normal_buffer_ids[i])
        glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, None)

        glEnableVertexAttribArray(0)
        glEnableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        # draw
        glDrawArrays(GL_TRIANGLES, 0, obj.geom_nums[i])

        glDisableVertexAttribArray(0)
        glDisableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        glBindVertexArray(0)
    # flower3d
    obj = flowerObj
    ModelMatrix = scale(0.05)
    MVP = dots(ProjectionMatrix, ViewMatrix, ModelMatrix)
    glUniformMatrix4fv(uniform_loc['M'], 1, GL_TRUE, c_matrix(ModelMatrix))
    glUniformMatrix4fv(uniform_loc['MVP'], 1, GL_TRUE, c_matrix(MVP))
    for i in range(len(obj.geom_nums)):
        glBindVertexArray(obj.vao_ids[i])
        # mtl parameter
        mtl = obj.mtl_buffer_list[i]
        glUniform3f(uniform_loc['material.ambient'], mtl.Ka[0], mtl.Ka[1], mtl.Ka[2])
        glUniform3f(uniform_loc['material.diffuse'], mtl.Kd[0], mtl.Kd[1], mtl.Kd[2])
        glUniform3f(uniform_loc['material.specular'], mtl.Ks[0], mtl.Ks[1], mtl.Ks[2])

        # texture
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, obj.texture_ids[i])
        glUniform1i(uniform_loc['myTextureSampler'], 0)

        # vertex
        glBindBuffer(GL_ARRAY_BUFFER, obj.vertex_buffer_ids[i])
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, None)
        # uv
        glBindBuffer(GL_ARRAY_BUFFER, obj.uv_buffer_ids[i])
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, None)
        # normal
        glBindBuffer(GL_ARRAY_BUFFER, obj.normal_buffer_ids[i])
        glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, None)

        glEnableVertexAttribArray(0)
        glEnableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        # draw
        glDrawArrays(GL_TRIANGLES, 0, obj.geom_nums[i])

        glDisableVertexAttribArray(0)
        glDisableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        glBindVertexArray(0)

    # swap buffer
    glutSwapBuffers()

def update(val):
    if val == 1:
        glutPostRedisplay()
        # glutTimerFunc(int(1000 / FPS), update, 1)

def init_opengl():
    """set opengl parameter"""
    # depth test
    glEnable(GL_DEPTH_TEST)
    glDepthFunc(GL_LEQUAL)
    glEnable(GL_CULL_FACE)
    # glEnable(GL_LINE_SMOOTH)
    # glEnable(GL_BLEND)

    glClearColor(0.0, 0.0, 0.4, 0.0)

def init_texture():
    pass

def init_glut(argv):
    """glut initialization."""
    global window, g_camera
    glutInit(argv)
    glutInitWindowSize(WIDTH, HEIGHT)
    glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH | GLUT_MULTISAMPLE)

    print(glutGet(GLUT_WINDOW_NUM_SAMPLES))
    window = glutCreateWindow(b'flower')

    glutReshapeFunc(reshape)
    glutDisplayFunc(display)
    glutKeyboardFunc(g_camera.keyboard)
    glutMouseFunc(g_camera.mouse)
    glutMouseWheelFunc(g_camera.mouseWheel)
    glutMotionFunc(g_camera.motion)
    glutTimerFunc(int(1000 / FPS), update, 1)

def init_shader():
    """load shader and set initial value"""
    global program_id, uniforms, uniform_loc
    program_id = BaseShaderProgram(
        'shaders/v_light.glsl', 'shaders/f_light.glsl').program_id
    # 'shaders/v_simple_texture.glsl', 'shaders/f_simple_texture.glsl').program_id
    # 'shaders/Transform.vs.glsl', 'shaders/f_green.glsl').program_id

    for uniform in uniforms:
        uniform_loc[uniform] = glGetUniformLocation(program_id, uniform)

def init_object():
    """load object data"""
    # read obj file
    global boxObj,flowerObj
    boxObj = Object(os.path.join('..', 'resources', 'box.obj'))
    boxObj.bind_buffer()
    flowerObj = Object(os.path.join('..', 'resources', 'flower3d.obj'))
    flowerObj.bind_buffer()

def main(argv=None):
    if argv is None:
        argv = sys.argv
    init_glut(argv)
    init_opengl()
    init_shader()
    init_texture()
    init_object()
    glutMainLoop()

main()
<commit_msg>delete GL_CULL_FACE<commit_after>from OpenGL.GL import *
from OpenGL.GLUT import *
from shader import BaseShaderProgram
from object import Object
import os
import sys
import numpy as np
from matrix_transform import *
from math import pi, radians
from Camera import Camera

WIDTH = 800
HEIGHT = 600
FPS = 50
ESC = b'\033'
SIZE_OF_FLOAT = 4

window = 0
program_id = 0

# uniform variables
uniform_loc = {}
uniforms = ['myTextureSampler',
            'MVP', 'M', 'V', 'NormalMatrix',
            'lightPos_worldspace',
            'material.ambient',
            'material.diffuse',
            'material.specular',
            'material.shininess',
            ]

# motion control
g_camera = Camera()

lightPos_worldspace = np.array([0, 0, 3])

#Objects
boxObj = None
flowerObj = None
g_object_list = []

def reshape(w, h):
    """TODO"""
    glViewport(0, 0, w, h)

def display():
    global lightPos_worldspace, g_camera
    global boxObj, flowerObj
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

    glUseProgram(program_id)
    # set transform matrix
    # matrix_scale = scale(g_scale_ratio)
    # ModelMatrix = rotate(matrix_scale.T, g_verticalAngle, np.array([1, 0, 0], 'f'))
    # ModelMatrix = rotate(ModelMatrix.T, g_horizontalAngle, np.array([0, 1, 0], 'f'))
    # matrix_translate = translate(g_translate_x, g_translate_y, 0.0)
    # ModelMatrix = np.dot(matrix_translate, ModelMatrix)

    ProjectionMatrix = perspective(radians(g_camera.zoom), 4 / 3, 0.1, 100)
    ViewMatrix = g_camera.getViewMatrix()
    glUniformMatrix4fv(uniform_loc['V'], 1, GL_TRUE, c_matrix(ViewMatrix))

    # set light parameter
    lightPos = g_camera.getLightPosition()
    glUniform3f(uniform_loc['lightPos_worldspace'], lightPos[0], lightPos[1], lightPos[2])
    glUniform1f(uniform_loc['material.shininess'], 32)

    # display
    obj = boxObj
    ModelMatrix = np.identity(4, 'f')
    matrix_translate = translate(2, 0, -5)
    ModelMatrix = dots(matrix_translate, ModelMatrix)
    MVP = dots(ProjectionMatrix, ViewMatrix, ModelMatrix)
    glUniformMatrix4fv(uniform_loc['M'], 1, GL_TRUE, c_matrix(ModelMatrix))
    glUniformMatrix4fv(uniform_loc['MVP'], 1, GL_TRUE, c_matrix(MVP))
    for i in range(len(obj.geom_nums)):
        glBindVertexArray(obj.vao_ids[i])
        # mtl parameter
        mtl = obj.mtl_buffer_list[i]
        glUniform3f(uniform_loc['material.ambient'], mtl.Ka[0], mtl.Ka[1], mtl.Ka[2])
        glUniform3f(uniform_loc['material.diffuse'], mtl.Kd[0], mtl.Kd[1], mtl.Kd[2])
        glUniform3f(uniform_loc['material.specular'], mtl.Ks[0], mtl.Ks[1], mtl.Ks[2])

        # texture
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, obj.texture_ids[i])
        glUniform1i(uniform_loc['myTextureSampler'], 0)

        # vertex
        glBindBuffer(GL_ARRAY_BUFFER, obj.vertex_buffer_ids[i])
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, None)
        # uv
        glBindBuffer(GL_ARRAY_BUFFER, obj.uv_buffer_ids[i])
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, None)
        # normal
        glBindBuffer(GL_ARRAY_BUFFER, obj.normal_buffer_ids[i])
        glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, None)

        glEnableVertexAttribArray(0)
        glEnableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        # draw
        glDrawArrays(GL_TRIANGLES, 0, obj.geom_nums[i])

        glDisableVertexAttribArray(0)
        glDisableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        glBindVertexArray(0)
    # flower3d
    obj = flowerObj
    ModelMatrix = scale(0.05)
    MVP = dots(ProjectionMatrix, ViewMatrix, ModelMatrix)
    glUniformMatrix4fv(uniform_loc['M'], 1, GL_TRUE, c_matrix(ModelMatrix))
    glUniformMatrix4fv(uniform_loc['MVP'], 1, GL_TRUE, c_matrix(MVP))
    for i in range(len(obj.geom_nums)):
        glBindVertexArray(obj.vao_ids[i])
        # mtl parameter
        mtl = obj.mtl_buffer_list[i]
        glUniform3f(uniform_loc['material.ambient'], mtl.Ka[0], mtl.Ka[1], mtl.Ka[2])
        glUniform3f(uniform_loc['material.diffuse'], mtl.Kd[0], mtl.Kd[1], mtl.Kd[2])
        glUniform3f(uniform_loc['material.specular'], mtl.Ks[0], mtl.Ks[1], mtl.Ks[2])

        # texture
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, obj.texture_ids[i])
        glUniform1i(uniform_loc['myTextureSampler'], 0)

        # vertex
        glBindBuffer(GL_ARRAY_BUFFER, obj.vertex_buffer_ids[i])
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, None)
        # uv
        glBindBuffer(GL_ARRAY_BUFFER, obj.uv_buffer_ids[i])
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, None)
        # normal
        glBindBuffer(GL_ARRAY_BUFFER, obj.normal_buffer_ids[i])
        glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, None)

        glEnableVertexAttribArray(0)
        glEnableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        # draw
        glDrawArrays(GL_TRIANGLES, 0, obj.geom_nums[i])

        glDisableVertexAttribArray(0)
        glDisableVertexAttribArray(1)
        glEnableVertexAttribArray(2)
        glBindVertexArray(0)

    # swap buffer
    glutSwapBuffers()

def update(val):
    if val == 1:
        glutPostRedisplay()
        # glutTimerFunc(int(1000 / FPS), update, 1)

def init_opengl():
    """set opengl parameter"""
    # depth test
    glEnable(GL_DEPTH_TEST)
    glDepthFunc(GL_LEQUAL)
    # glEnable(GL_CULL_FACE)
    # glEnable(GL_LINE_SMOOTH)
    # glEnable(GL_BLEND)

    glClearColor(0.0, 0.0, 0.4, 0.0)

def init_texture():
    pass

def init_glut(argv):
    """glut initialization."""
    global window, g_camera
    glutInit(argv)
    glutInitWindowSize(WIDTH, HEIGHT)
    glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH | GLUT_MULTISAMPLE)

    print(glutGet(GLUT_WINDOW_NUM_SAMPLES))
    window = glutCreateWindow(b'flower')

    glutReshapeFunc(reshape)
    glutDisplayFunc(display)
    glutKeyboardFunc(g_camera.keyboard)
    glutMouseFunc(g_camera.mouse)
    glutMouseWheelFunc(g_camera.mouseWheel)
    glutMotionFunc(g_camera.motion)
    glutTimerFunc(int(1000 / FPS), update, 1)

def init_shader():
    """load shader and set initial value"""
    global program_id, uniforms, uniform_loc
    program_id = BaseShaderProgram(
        'shaders/v_light.glsl', 'shaders/f_light.glsl').program_id
    # 'shaders/v_simple_texture.glsl', 'shaders/f_simple_texture.glsl').program_id
    # 'shaders/Transform.vs.glsl', 'shaders/f_green.glsl').program_id

    for uniform in uniforms:
        uniform_loc[uniform] = glGetUniformLocation(program_id, uniform)

def init_object():
    """load object data"""
    # read obj file
    global boxObj,flowerObj
    boxObj = Object(os.path.join('..', 'resources', 'box.obj'))
    boxObj.bind_buffer()
    flowerObj = Object(os.path.join('..', 'resources', 'flower3d.obj'))
    flowerObj.bind_buffer()

def main(argv=None):
    if argv is None:
        argv = sys.argv
    init_glut(argv)
    init_opengl()
    init_shader()
    init_texture()
    init_object()
    glutMainLoop()

main()
<|endoftext|>


<commit_before># encoding=utf-8
from typing import List
import numpy as np
import xgboost as xgb
import logging
import re
import tensorflow as tf
import tensorflow_addons as tfa

def build_xgb_nn_model(tree_num=10):
    """
    use xgb leaf features inside.
    To have a different model space in moe, just use xgb leaf features here.
    :return:
    """
    inputs = []

    embeddings = []
    for i in range(tree_num):
        inputs.append(tf.keras.Input(shape=(1,), name="tree_" + str(i)))
        embeddings.append(
            tf.keras.layers.Embedding(128, 32)(inputs[-1])
        )
    # leaf = tf.keras.Input(shape=(10,), name="tree_leaf")
    # inputs.append(leaf)
    leaf_context = tf.keras.layers.concatenate(embeddings, axis=-1)
    flatten_context = tf.keras.layers.Flatten()(leaf_context)
    fusion_context = tf.keras.layers.Dense(32, 'relu')(flatten_context)
    output = tf.keras.layers.Dense(1, 'sigmoid')(fusion_context)

    model = tf.keras.models.Model(inputs=inputs, outputs=[output], name='xgb_nn')
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[
        'accuracy',
        tf.keras.metrics.Precision(),
        tf.keras.metrics.Recall(),
        tf.keras.metrics.AUC(),
        tfa.metrics.F1Score(num_classes=1, threshold=0.5)
    ])
    model.summary()
    return model
class XGBLeafDataSource(object):
    """
    read the files and parse out the "app_rate" feature and use the xgb to get the leafs.
    """
    xgb_modelfile = 'xgb.fornn.model'
    xgb_param_dist = {'objective': 'binary:logistic', 'n_estimators': 10, 'max_depth': 6, 'subsample': 0.8,
                      'learning_rate': 0.1}
    xgbclf = xgb.XGBClassifier(**xgb_param_dist)
    try:
        xgbclf.load_model(xgb_modelfile)
        logging.info('load xgb model ok')
    except:
        pass

    def __init__(self, path, batch_size, span):
        self.path = path
        self.batch_size = batch_size
        self.span = span # feature span
        pass
    def iter_test(self):
        while True:
            # features_size = 406
            features, label = np.random.random((self.batch_size, 406)), np.random.randint(0, 2, (self.batch_size, 1))
            # print(label)
            yield {'app_rate': features}, label

    def iter(self, epoch=None):
        k = 0
        s, e = self.span
        while True:
            # print(k)
            if k == epoch: break
            k += 1

            with open(self.path, 'r') as f:
                batch = []
                ys = []
                for line in f:
                    us = line.split(" ")
                    y = int(us[1])
                    app_rate = [float(i) for i in us[s:e]]
                    # print(app_rate)
                    batch.append(app_rate)
                    ys.append(y)
                    if len(batch) == self.batch_size:
                        leafs = XGBLeafDataSource.xgbclf.apply(np.array(batch))

                        fs = {}
                        t = 0
                        for tree_leaf in np.transpose(leafs):
                            fs['tree_' + str(t)] =  tree_leaf
                            t += 1
                        yield fs, np.array(ys)
                        batch = []
                        ys = []

def xgbnn_train():
    train_path = "./data_store/train_data"
    valid_path = "./data_store/valid_data"

    model = build_xgb_nn_model(10)
    model_path = './xgbnn_model/weights'
    csvfile = './xgbnn.csv'
    submit = './submit.xgbnn.csv'

    batch_size = 10
    phone_source = XGBLeafDataSource(train_path, batch_size, (47, 453))
    valid_phone_source = XGBLeafDataSource(valid_path, batch_size, (47, 453))

    model.fit(phone_source.iter(),
              validation_data=valid_phone_source.iter(1),
              steps_per_epoch=100, epochs=20)
    model.save_weights(model_path)

def main():
    train_path = "./data_store/train_data"
    valid_path = "./data_store/valid_data"
    test_path = "./data_store/test_data"
    batch_size = 10
    phone_source = XGBLeafDataSource(train_path, batch_size, (47, 453))

    for leafs in phone_source.iter():
        # pass
        # leafs = xgbclf.apply(features['app_rate'])
        print(type(leafs))
        print(leafs)
        input("Press any key .. ")

    pass
if __name__ == '__main__':
    xgbnn_train()
    # main()
<commit_msg>Update phone_xgbnn.py<commit_after># encoding=utf-8
from typing import List
import numpy as np
import xgboost as xgb
import logging
import re
import tensorflow as tf
import tensorflow_addons as tfa

def build_xgb_nn_model(tree_num=10):
    """
    use xgb leaf features inside.
    To have a different model space in moe, just use xgb leaf features here.
    :return:
    """
    inputs = []

    embeddings = []
    for i in range(tree_num):
        inputs.append(tf.keras.Input(shape=(1,), name="tree_" + str(i)))
        embeddings.append(
            tf.keras.layers.Embedding(128, 32)(inputs[-1])
        )
    # leaf = tf.keras.Input(shape=(10,), name="tree_leaf")
    # inputs.append(leaf)
    leaf_context = tf.keras.layers.concatenate(embeddings, axis=-1)
    flatten_context = tf.keras.layers.Flatten()(leaf_context)
    fusion_context = tf.keras.layers.Dense(32, 'relu')(flatten_context)
    output = tf.keras.layers.Dense(1, 'sigmoid')(fusion_context)

    model = tf.keras.models.Model(inputs=inputs, outputs=[output], name='xgb_nn')
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[
        'accuracy',
        tf.keras.metrics.Precision(),
        tf.keras.metrics.Recall(),
        tf.keras.metrics.AUC(),
        tfa.metrics.F1Score(num_classes=1, threshold=0.5)
    ])
    model.summary()
    return model
class XGBLeafDataSource(object):
    """
    read the files and parse out the "app_rate" feature and use the xgb to get the leafs.
    """
    xgb_modelfile = 'xgb.fornn.model'
    xgb_param_dist = {'objective': 'binary:logistic', 'n_estimators': 10, 'max_depth': 6, 'subsample': 0.8,
                      'learning_rate': 0.1}
    xgbclf = xgb.XGBClassifier(**xgb_param_dist)
    try:
        xgbclf.load_model(xgb_modelfile)
        logging.info('load xgb model ok')
    except:
        pass

    def __init__(self, path, batch_size, span):
        self.path = path
        self.batch_size = batch_size
        self.span = span # feature span
        pass
    def iter_test(self):
        while True:
            # features_size = 406
            features, label = np.random.random((self.batch_size, 406)), np.random.randint(0, 2, (self.batch_size, 1))
            # print(label)
            yield {'app_rate': features}, label

    def iter(self, epoch=None):
        k = 0
        s, e = self.span
        while True:
            # print(k)
            if k == epoch:
                # k = 0
                break
            k += 1

            with open(self.path, 'r') as f:
                batch = []
                ys = []
                for line in f:
                    us = line.split(" ")
                    y = int(us[1])
                    app_rate = [float(i) for i in us[s:e]]
                    # print(app_rate)
                    batch.append(app_rate)
                    ys.append(y)
                    if len(batch) == self.batch_size:
                        leafs = XGBLeafDataSource.xgbclf.apply(np.array(batch))

                        fs = {}
                        t = 0
                        for tree_leaf in np.transpose(leafs):
                            fs['tree_' + str(t)] =  tree_leaf
                            t += 1
                        yield fs, np.array(ys)
                        batch = []
                        ys = []

def xgbnn_train():
    train_path = "./data_store/train_data"
    valid_path = "./data_store/valid_data"

    model = build_xgb_nn_model(10)
    model_path = './xgbnn_model/weights'
    csvfile = './xgbnn.csv'
    submit = './submit.xgbnn.csv'

    batch_size = 10
    phone_source = XGBLeafDataSource(train_path, batch_size, (47, 453))
    valid_phone_source = XGBLeafDataSource(valid_path, batch_size, (47, 453))

    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath=model_path,
        save_weights_only=True,
        monitor='val_f1_score',
        mode='max',
        save_best_only=True)

    csvlogger = tf.keras.callbacks.CSVLogger(filename=csvfile)

    model.fit(phone_source.iter(),
              validation_data=valid_phone_source.iter(),
              steps_per_epoch=100, epochs=20,
              validation_steps=20,
              callbacks=[model_checkpoint_callback,csvlogger ]
              )
    model.save_weights(model_path)

def main():
    train_path = "./data_store/train_data"
    valid_path = "./data_store/valid_data"
    test_path = "./data_store/test_data"
    batch_size = 10
    phone_source = XGBLeafDataSource(train_path, batch_size, (47, 453))

    for leafs in phone_source.iter(1):
        # pass
        # leafs = xgbclf.apply(features['app_rate'])
        print(type(leafs))
        print(leafs)
        input("Press any key .. ")

    pass
if __name__ == '__main__':
    xgbnn_train()
    # main()
<|endoftext|>


<commit_before>import numpy
import numpy.random
import scipy.sparse

def diag_dot(a, b, out=None):
    """Input:  a and b are arrays.
       Output: a column vector of the dot product of the rows of a and respective
       columns of b, in other words, diag(a.dot(b))."""
    if out is None:
        out = numpy.empty((a.shape[0], 1))
    numpy.einsum('ji,ij->j', a, b, out=out[:,0])
    return out

class NeuralLM(object):
    def __init__(self, ngram_size, input_vocab_size, output_vocab_size, input_embedding_dimension, num_hidden, output_embedding_dimension, activation_function):
        self.input_vocab_size = input_vocab_size
        self.output_vocab_size = output_vocab_size
        self.index_to_word = []
        self.index_to_word_input = []
        self.index_to_word_output = []
        self.word_to_index = {}
        self.word_to_index_input = {}
        self.word_to_index_output = {}

        self.ngram_size = ngram_size
        self.input_embedding_dimension = input_embedding_dimension
        self.num_hidden = num_hidden
        self.output_embedding_dimension = output_embedding_dimension
        self.activation_function = activation_function

        self.input_embeddings = numpy.zeros((input_vocab_size,      input_embedding_dimension))
        if not num_hidden:
            self.hidden1_weights  = numpy.zeros((output_embedding_dimension,            (ngram_size-1)*input_embedding_dimension))
            self.hidden2_weights  = numpy.zeros((1, 1))
        else:
            self.hidden1_weights  = numpy.zeros((num_hidden,            (ngram_size-1)*input_embedding_dimension))
            self.hidden2_weights  = numpy.zeros((output_embedding_dimension, num_hidden))
        self.output_weights   = numpy.zeros((output_vocab_size,     output_embedding_dimension))
        self.output_biases    = numpy.zeros((output_vocab_size,     1))

    def initialize(self, r):
        def uniform(m):
            m[:,:] = numpy.random.uniform(-r, r, m.shape)
        uniform(self.input_embeddings)
        uniform(self.hidden1_weights)
        uniform(self.hidden2_weights)
        uniform(self.output_weights)
        uniform(self.output_biases)

    def forward_prop(self, inputs, output=None, normalize=True):
        u = numpy.bmat([[self.input_embeddings.T * ui] for ui in inputs])
        h1 = numpy.maximum(0., self.hidden1_weights * u)
        if not self.num_hidden:
            h2 = h1
        else:
            h2 = numpy.maximum(0., self.hidden2_weights * h1)

        if output is None:
            o = self.output_weights * h2 + self.output_biases
        else:
            # Inefficient version:
            #o = diag_dot(output.T, (self.output_weights * h2 + self.output_biases))
            #o = output.multiply(self.output_weights * h2 + self.output_biases)

            # Since output is sparse, distributing multiplication by output
            # is much more efficient:

            o = diag_dot(output.T * self.output_weights, h2) + output.T * self.output_biases
        return o

    def backward_prop(self, g_output):
        pass

    def to_file(self, outfile):

        def write_matrix(m):
            for i in xrange(m.shape[0]):
                outfile.write("\t".join(map(str, m[i])))
                outfile.write("\n")
            outfile.write("\n")

        def write_vector(m):
            for i in xrange(m.shape[0]):
                outfile.write(str(m[i]))
                outfile.write("\n")
            outfile.write("\n")

        outfile.write("\\config\n")
        outfile.write("version 1\n")
        outfile.write("ngram_size %d\n" % self.ngram_size)
        outfile.write("input_vocab_size %d\n" % self.input_vocab_size)
        outfile.write("output_vocab_size %d\n" % self.output_vocab_size)
        outfile.write("input_embedding_dimension %d\n" % self.input_embedding_dimension)
        outfile.write("num_hidden %d\n" % self.num_hidden)
        outfile.write("output_embedding_dimension %d\n" % self.output_embedding_dimension)
        outfile.write("activation_function %s\n" % self.activation_function)
        outfile.write("\n")

        if self.index_to_word_input and self.index_to_word_output:
            outfile.write("\\input_vocab\n")
            for word in self.index_to_word_input:
                outfile.write(word + "\n")
            outfile.write("\n")

            outfile.write("\\output_vocab\n")
            for word in self.index_to_word_output:
                outfile.write(word + "\n")
            outfile.write("\n")

        elif self.index_to_word:
            outfile.write("\\vocab\n")
            for word in self.index_to_word:
                outfile.write(word + "\n")
            outfile.write("\n")

        outfile.write("\\input_embeddings\n")
        write_matrix(self.input_embeddings)

        outfile.write("\\hidden_weights 1\n")
        write_matrix(self.hidden1_weights)

        outfile.write("\\hidden_weights 2\n")
        write_matrix(self.hidden2_weights)

        outfile.write("\\output_weights\n")
        write_matrix(self.output_weights)

        outfile.write("\\output_biases\n")
        write_matrix(self.output_biases)

        outfile.write("\\end\n")

    @staticmethod
    def from_file(infile):
        """Create a NeuralLM from a text file."""

        # Helper functions
        def read_sections(infile):
            while True:
                line = infile.next().strip()
                if line == "\\end":
                    break
                elif line.startswith('\\'):
                    yield line, read_section(infile)

        def read_section(infile):
            while True:
                line = infile.next().strip()
                if line == "":
                    break
                else:
                    yield line

        def read_matrix(lines, m, n, out=None):
            if out is None:
                out = numpy.zeros((m, n))
            i = 0
            for line in lines:
                row = numpy.array(map(float, line.split()))
                if len(row) != n:
                    raise Exception("wrong number of columns (expected %d, found %d)" % (n, len(row)))
                if i >= m:
                    raise Exception("wrong number of rows (expected %d, found more)" % m)
                out[i,:] = row
                i += 1
            if i < m:
                raise Exception("wrong number of rows (expected %d, found %d)" % (m, i))
            return out

        if isinstance(infile, str):
            infile = open(infile)

        for section, lines in read_sections(infile):
            if section == "\\config":
                config = {}
                for line in lines:
                    key, value = line.split()
                    config[key] = value

                m = NeuralLM(ngram_size=int(config['ngram_size']),
                             input_vocab_size=int(config['input_vocab_size']),
                             output_vocab_size=int(config['output_vocab_size']),
                             input_embedding_dimension=int(config['input_embedding_dimension']),
                             num_hidden=int(config['num_hidden']),
                             output_embedding_dimension=int(config['output_embedding_dimension']),
                             activation_function=config['activation_function'])

            elif section == "\\input_vocab":
                for line in lines:
                    m.index_to_word_input.append(line)
                m.word_to_index_input = dict((w,i) for (i,w) in enumerate(m.index_to_word_input))

            elif section == "\\output_vocab":
                for line in lines:
                    m.index_to_word_output.append(line)
                m.word_to_index_output = dict((w,i) for (i,w) in enumerate(m.index_to_word_output))

            elif section == "\\vocab":
                for line in lines:
                    m.index_to_word.append(line)
                m.word_to_index = dict((w,i) for (i,w) in enumerate(m.index_to_word))

            elif section == "\\input_embeddings":
                read_matrix(lines, m.input_vocab_size, m.input_embedding_dimension, out=m.input_embeddings)
            elif section == "\\hidden_weights 1":
                if not m.num_hidden:
                    read_matrix(lines, m.output_embedding_dimension, (m.ngram_size-1)*m.input_embedding_dimension, out=m.hidden1_weights)
                else:
                    read_matrix(lines, m.num_hidden, (m.ngram_size-1)*m.input_embedding_dimension, out=m.hidden1_weights)
            elif section == "\\hidden_weights 2":
                if not m.num_hidden:
                    read_matrix(lines, 1, 1, out=m.hidden2_weights)
                else:
                    read_matrix(lines, m.output_embedding_dimension, m.num_hidden, out=m.hidden2_weights)
            elif section == "\\output_weights":
                read_matrix(lines, m.output_vocab_size, m.output_embedding_dimension, out=m.output_weights)
            elif section == "\\output_biases":
                read_matrix(lines, m.output_vocab_size, 1, out=m.output_biases)
        return m

    def make_data(self, ngrams):
        """Takes a list of n-grams of words (as ints),
           and converts into a list of n sparse arrays."""
        rows = [[] for j in xrange(self.ngram_size)]
        cols = [[] for j in xrange(self.ngram_size)]
        values = [[] for j in xrange(self.ngram_size)]
        for i, ngram in enumerate(ngrams):
            for j, w in enumerate(ngram):
                rows[j].append(w)
                cols[j].append(i)
                values[j].append(1)
        data = [scipy.sparse.csc_matrix((values[j], (rows[j], cols[j])), shape=(self.input_vocab_size, len(ngrams))) for j in xrange(self.ngram_size-1)]
        data.append(scipy.sparse.csc_matrix((values[-1], (rows[-1], cols[-1])), shape=(self.output_vocab_size, len(ngrams))))
        return data
<commit_msg>python: support hidden biases<commit_after>import numpy
import numpy.random
import scipy.sparse

def diag_dot(a, b, out=None):
    """Input:  a and b are arrays.
       Output: a column vector of the dot product of the rows of a and respective
       columns of b, in other words, diag(a.dot(b))."""
    if out is None:
        out = numpy.empty((a.shape[0], 1))
    numpy.einsum('ji,ij->j', a, b, out=out[:,0])
    return out

class NeuralLM(object):
    def __init__(self, ngram_size, input_vocab_size, output_vocab_size, input_embedding_dimension, num_hidden, output_embedding_dimension, activation_function):
        self.input_vocab_size = input_vocab_size
        self.output_vocab_size = output_vocab_size
        self.index_to_word = []
        self.index_to_word_input = []
        self.index_to_word_output = []
        self.word_to_index = {}
        self.word_to_index_input = {}
        self.word_to_index_output = {}

        self.ngram_size = ngram_size
        self.input_embedding_dimension = input_embedding_dimension
        self.num_hidden = num_hidden
        self.output_embedding_dimension = output_embedding_dimension
        self.activation_function = activation_function

        self.input_embeddings = numpy.zeros((input_vocab_size,      input_embedding_dimension))
        if not num_hidden:
            self.hidden1_weights  = numpy.zeros((output_embedding_dimension,            (ngram_size-1)*input_embedding_dimension))
            self.hidden1_biases  = numpy.zeros((output_embedding_dimension,             1))
            self.hidden2_weights  = numpy.zeros((1, 1))
            self.hidden2_biases  = numpy.zeros((1, 1))
        else:
            self.hidden1_weights  = numpy.zeros((num_hidden,            (ngram_size-1)*input_embedding_dimension))
            self.hidden1_biases  = numpy.zeros((num_hidden,            1))
            self.hidden2_weights  = numpy.zeros((output_embedding_dimension, num_hidden))
            self.hidden2_biases  = numpy.zeros((output_embedding_dimension, 1))
        self.output_weights   = numpy.zeros((output_vocab_size,     output_embedding_dimension))
        self.output_biases    = numpy.zeros((output_vocab_size,     1))

    def initialize(self, r):
        def uniform(m):
            m[:,:] = numpy.random.uniform(-r, r, m.shape)
        uniform(self.input_embeddings)
        uniform(self.hidden1_weights)
        uniform(self.hidden1_biases)
        uniform(self.hidden2_weights)
        uniform(self.hidden2_biases)
        uniform(self.output_weights)
        uniform(self.output_biases)

    def forward_prop(self, inputs, output=None, normalize=True):
        u = numpy.bmat([[self.input_embeddings.T * ui] for ui in inputs])
        h1 = numpy.maximum(0., self.hidden1_weights * u + self.hidden1_biases)
        if not self.num_hidden:
            h2 = h1
        else:
            h2 = numpy.maximum(0., self.hidden2_weights * h1 + self.hidden2_biases)

        if output is None:
            o = self.output_weights * h2 + self.output_biases
        else:
            # Inefficient version:
            #o = diag_dot(output.T, (self.output_weights * h2 + self.output_biases))
            #o = output.multiply(self.output_weights * h2 + self.output_biases)

            # Since output is sparse, distributing multiplication by output
            # is much more efficient:

            o = diag_dot(output.T * self.output_weights, h2) + output.T * self.output_biases
        return o

    def backward_prop(self, g_output):
        pass

    def to_file(self, outfile):

        def write_matrix(m):
            for i in xrange(m.shape[0]):
                outfile.write("\t".join(map(str, m[i])))
                outfile.write("\n")
            outfile.write("\n")

        def write_vector(m):
            for i in xrange(m.shape[0]):
                outfile.write(str(m[i]))
                outfile.write("\n")
            outfile.write("\n")

        outfile.write("\\config\n")
        outfile.write("version 1\n")
        outfile.write("ngram_size %d\n" % self.ngram_size)
        outfile.write("input_vocab_size %d\n" % self.input_vocab_size)
        outfile.write("output_vocab_size %d\n" % self.output_vocab_size)
        outfile.write("input_embedding_dimension %d\n" % self.input_embedding_dimension)
        outfile.write("num_hidden %d\n" % self.num_hidden)
        outfile.write("output_embedding_dimension %d\n" % self.output_embedding_dimension)
        outfile.write("activation_function %s\n" % self.activation_function)
        outfile.write("\n")

        if self.index_to_word_input and self.index_to_word_output:
            outfile.write("\\input_vocab\n")
            for word in self.index_to_word_input:
                outfile.write(word + "\n")
            outfile.write("\n")

            outfile.write("\\output_vocab\n")
            for word in self.index_to_word_output:
                outfile.write(word + "\n")
            outfile.write("\n")

        elif self.index_to_word:
            outfile.write("\\vocab\n")
            for word in self.index_to_word:
                outfile.write(word + "\n")
            outfile.write("\n")

        outfile.write("\\input_embeddings\n")
        write_matrix(self.input_embeddings)

        outfile.write("\\hidden_weights 1\n")
        write_matrix(self.hidden1_weights)

        outfile.write("\\hidden_biases 1\n")
        write_matrix(self.hidden1_biases)

        outfile.write("\\hidden_weights 2\n")
        write_matrix(self.hidden2_weights)

        outfile.write("\\hidden_biases 2\n")
        write_matrix(self.hidden2_biases)

        outfile.write("\\output_weights\n")
        write_matrix(self.output_weights)

        outfile.write("\\output_biases\n")
        write_matrix(self.output_biases)

        outfile.write("\\end\n")

    @staticmethod
    def from_file(infile):
        """Create a NeuralLM from a text file."""

        # Helper functions
        def read_sections(infile):
            while True:
                line = infile.next().strip()
                if line == "\\end":
                    break
                elif line.startswith('\\'):
                    yield line, read_section(infile)

        def read_section(infile):
            while True:
                line = infile.next().strip()
                if line == "":
                    break
                else:
                    yield line

        def read_matrix(lines, m, n, out=None):
            if out is None:
                out = numpy.zeros((m, n))
            i = 0
            for line in lines:
                row = numpy.array(map(float, line.split()))
                if len(row) != n:
                    raise Exception("wrong number of columns (expected %d, found %d)" % (n, len(row)))
                if i >= m:
                    raise Exception("wrong number of rows (expected %d, found more)" % m)
                out[i,:] = row
                i += 1
            if i < m:
                raise Exception("wrong number of rows (expected %d, found %d)" % (m, i))
            return out

        if isinstance(infile, str):
            infile = open(infile)

        for section, lines in read_sections(infile):
            if section == "\\config":
                config = {}
                for line in lines:
                    key, value = line.split()
                    config[key] = value

                m = NeuralLM(ngram_size=int(config['ngram_size']),
                             input_vocab_size=int(config['input_vocab_size']),
                             output_vocab_size=int(config['output_vocab_size']),
                             input_embedding_dimension=int(config['input_embedding_dimension']),
                             num_hidden=int(config['num_hidden']),
                             output_embedding_dimension=int(config['output_embedding_dimension']),
                             activation_function=config['activation_function'])

            elif section == "\\input_vocab":
                for line in lines:
                    m.index_to_word_input.append(line)
                m.word_to_index_input = dict((w,i) for (i,w) in enumerate(m.index_to_word_input))

            elif section == "\\output_vocab":
                for line in lines:
                    m.index_to_word_output.append(line)
                m.word_to_index_output = dict((w,i) for (i,w) in enumerate(m.index_to_word_output))

            elif section == "\\vocab":
                for line in lines:
                    m.index_to_word.append(line)
                m.word_to_index = dict((w,i) for (i,w) in enumerate(m.index_to_word))

            elif section == "\\input_embeddings":
                read_matrix(lines, m.input_vocab_size, m.input_embedding_dimension, out=m.input_embeddings)
            elif section == "\\hidden_weights 1":
                if not m.num_hidden:
                    read_matrix(lines, m.output_embedding_dimension, (m.ngram_size-1)*m.input_embedding_dimension, out=m.hidden1_weights)
                else:
                    read_matrix(lines, m.num_hidden, (m.ngram_size-1)*m.input_embedding_dimension, out=m.hidden1_weights)
            elif section == "\\hidden_biases 1":
                if not m.num_hidden:
                    read_matrix(lines, m.output_embedding_dimension, 1, out=m.hidden1_biases)
                else:
                    read_matrix(lines, m.num_hidden, 1, out=m.hidden1_biases)
            elif section == "\\hidden_weights 2":
                if not m.num_hidden:
                    read_matrix(lines, 1, 1, out=m.hidden2_weights)
                else:
                    read_matrix(lines, m.output_embedding_dimension, m.num_hidden, out=m.hidden2_weights)
            elif section == "\\hidden_biases 2":
                if not m.num_hidden:
                    read_matrix(lines, 1, 1, out=m.hidden2_biases)
                else:
                    read_matrix(lines, m.output_embedding_dimension, 1, out=m.hidden2_biases)
            elif section == "\\output_weights":
                read_matrix(lines, m.output_vocab_size, m.output_embedding_dimension, out=m.output_weights)
            elif section == "\\output_biases":
                read_matrix(lines, m.output_vocab_size, 1, out=m.output_biases)
        return m

    def make_data(self, ngrams):
        """Takes a list of n-grams of words (as ints),
           and converts into a list of n sparse arrays."""
        rows = [[] for j in xrange(self.ngram_size)]
        cols = [[] for j in xrange(self.ngram_size)]
        values = [[] for j in xrange(self.ngram_size)]
        for i, ngram in enumerate(ngrams):
            for j, w in enumerate(ngram):
                rows[j].append(w)
                cols[j].append(i)
                values[j].append(1)
        data = [scipy.sparse.csc_matrix((values[j], (rows[j], cols[j])), shape=(self.input_vocab_size, len(ngrams))) for j in xrange(self.ngram_size-1)]
        data.append(scipy.sparse.csc_matrix((values[-1], (rows[-1], cols[-1])), shape=(self.output_vocab_size, len(ngrams))))
        return data
<|endoftext|>