<commit_before>import os
import shutil
from flask import Flask, render_template, request, flash, redirect, url_for, session
from flask_mail import Mail, Message
from celery import Celery

# initialize flask app and configs
app = Flask(__name__)
app.config.from_object('config')

APP_ROOT = os.path.join(os.path.dirname(__file__))
print("APP_ROOT is {0}".format(APP_ROOT))
print("listdir(APP_ROOT) is {0}".format(os.listdir(APP_ROOT)))

app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

# start up celery
celery = Celery(app.name, broker=os.environ.get('REDISTOGO_URL'))
celery.conf.update(app.config)
# for heroku w/celery and RedisToGo add-on
broker_url = os.environ.get('REDISTOGO_URL')
celery.conf.update(BROKER_URL=broker_url,
                   CELERY_RESULT_BACKEND=broker_url)

# initialize mail
mail = Mail(app)

# initialize flickr object
import flickrapi, json, urllib
from ratelimit import rate_limited
flickr_key = os.environ.get('FLICKR_API_KEY')
flickr_secret = os.environ.get('FLICKR_API_SECRET')
flickr = flickrapi.FlickrAPI(flickr_key, flickr_secret, format='parsed-json')

# ratelimit each hit against flickrapi
@rate_limited(1)
def initialize_silo(tag, per_page, page):
    return flickr.photos.search(tags=tag, per_page=per_page, page=page)

@rate_limited(1)
def get_photo_sizes(photo_id):
    return flickr.photos.getSizes(photo_id=photo_id)

# create bucket directory if necessary
def find_bucket(path):
    if not os.path.exists(path):
        os.mkdir(path)

# return a page of flickr photos as JSON
def get_page(tag, page=1):
    # silo we fetch food from, per_page max = 500
    silo = initialize_silo(tag, 500, page)
    food = silo['photos']['photo']
    return food

# loop through photos, pick and download best size for each
def fill_bucket(food, path):
    for f in food:
        options = get_photo_sizes(f['id'])
        best = None
        for option in options['sizes']['size']:
            best = option['source']
            if option['label'] == 'Original':
                break
        if best:
            name = ''.join(e for e in f['title'] if e.isalnum()) + f['id']
            # sometimes names are too long so we slice 'em up
            name = '.'.join([name[:100], 'jpg'])
            urllib.request.urlretrieve(best, os.path.join(path, name))

# get buckets of food
def fetch_buckets(tag, path, n=1):
    silo = initialize_silo(tag, 5, 1)
    total = silo['photos']['pages']
    if n > total or n <= 0:
        n = total
    for i in range(0, n):
        food = get_page(tag, i)
        fill_bucket(food, path)

# feedingtube main function
@celery.task
def get_food(email, tag):
    with app.app_context():
        # nav to appropriate directory
        clean_tag = ''.join(tag.split())
        path = os.path.join(APP_ROOT, 'foodstuff', ''.join([email.split("@")[0], clean_tag]))
        find_bucket(path)
        # fill with images
        fetch_buckets(tag, path, 1)
        # zip directory contents
        shutil.make_archive(clean_tag, 'zip', path)
        # build the email
        msg = Message(subject='Dinner\'s ready!',
                      sender='no-reply@feedingtube.host',
                      recipients=[email])
        msg.body = 'Your images for {0} are attached as a zip file.'.format(tag)
        # attach the zipfile to the email
        zipfile = '.'.join([clean_tag, 'zip'])
        os.chdir(path)
        with app.open_resource(zipfile) as z:
            msg.attach(filename=zipfile, content_type="archive/zip", data=z.read())
        # send the email
        os.chdir(APP_ROOT)
        mail.send(msg)
        # wipe path
        shutil.rmtree(path)
        # after rmtree, zip files bubble up -- kill 'em!
        if os.path.exists(zipfile):
            os.remove(zipfile)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'GET':
        return render_template('index.html', email=session.get('email', ''), tag=session.get('tag', ''))
    email = request.form['email']
    tag = request.form['tag']
    session['email'] = email
    session['tag'] = tag
    get_food.apply_async(args=[email, tag])
    flash('Sometimes this part takes a while. We\'ll send it all over to {0} when it\'s ready. Thanks for being patient!'.format(email))
    return render_template('index.html', email=session.get('email', ''), tag=session.get('tag', ''))

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Ensure foodstuff directory exists<commit_after>import os
import shutil
from flask import Flask, render_template, request, flash, redirect, url_for, session
from flask_mail import Mail, Message
from celery import Celery

# initialize flask app and configs
app = Flask(__name__)
app.config.from_object('config')

APP_ROOT = os.path.join(os.path.dirname(__file__))

app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

# start up celery
celery = Celery(app.name, broker=os.environ.get('REDISTOGO_URL'))
celery.conf.update(app.config)
# for heroku w/celery and RedisToGo add-on
broker_url = os.environ.get('REDISTOGO_URL')
celery.conf.update(BROKER_URL=broker_url,
                   CELERY_RESULT_BACKEND=broker_url)

# initialize mail
mail = Mail(app)

# initialize flickr object
import flickrapi, json, urllib
from ratelimit import rate_limited
flickr_key = os.environ.get('FLICKR_API_KEY')
flickr_secret = os.environ.get('FLICKR_API_SECRET')
flickr = flickrapi.FlickrAPI(flickr_key, flickr_secret, format='parsed-json')

# ratelimit each hit against flickrapi
@rate_limited(1)
def initialize_silo(tag, per_page, page):
    return flickr.photos.search(tags=tag, per_page=per_page, page=page)

@rate_limited(1)
def get_photo_sizes(photo_id):
    return flickr.photos.getSizes(photo_id=photo_id)

# create bucket directory if necessary
def find_bucket(path):
    if not os.path.exists(os.path.join(APP_ROOT, 'foodstuff')):
        os.mkdir(os.path.join(APP_ROOT, 'foodstuff'))
    if not os.path.exists(path):
        os.mkdir(path)

# return a page of flickr photos as JSON
def get_page(tag, page=1):
    # silo we fetch food from, per_page max = 500
    silo = initialize_silo(tag, 500, page)
    food = silo['photos']['photo']
    return food

# loop through photos, pick and download best size for each
def fill_bucket(food, path):
    for f in food:
        options = get_photo_sizes(f['id'])
        best = None
        for option in options['sizes']['size']:
            best = option['source']
            if option['label'] == 'Original':
                break
        if best:
            name = ''.join(e for e in f['title'] if e.isalnum()) + f['id']
            # sometimes names are too long so we slice 'em up
            name = '.'.join([name[:100], 'jpg'])
            urllib.request.urlretrieve(best, os.path.join(path, name))

# get buckets of food
def fetch_buckets(tag, path, n=1):
    silo = initialize_silo(tag, 5, 1)
    total = silo['photos']['pages']
    if n > total or n <= 0:
        n = total
    for i in range(0, n):
        food = get_page(tag, i)
        fill_bucket(food, path)

# feedingtube main function
@celery.task
def get_food(email, tag):
    with app.app_context():
        # nav to appropriate directory
        clean_tag = ''.join(tag.split())
        path = os.path.join(APP_ROOT, 'foodstuff', ''.join([email.split("@")[0], clean_tag]))
        find_bucket(path)
        # fill with images
        fetch_buckets(tag, path, 1)
        # zip directory contents
        shutil.make_archive(clean_tag, 'zip', path)
        # build the email
        msg = Message(subject='Dinner\'s ready!',
                      sender='no-reply@feedingtube.host',
                      recipients=[email])
        msg.body = 'Your images for {0} are attached as a zip file.'.format(tag)
        # attach the zipfile to the email
        zipfile = '.'.join([clean_tag, 'zip'])
        os.chdir(path)
        with app.open_resource(zipfile) as z:
            msg.attach(filename=zipfile, content_type="archive/zip", data=z.read())
        # send the email
        os.chdir(APP_ROOT)
        mail.send(msg)
        # wipe path
        shutil.rmtree(path)
        # after rmtree, zip files bubble up -- kill 'em!
        if os.path.exists(zipfile):
            os.remove(zipfile)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'GET':
        return render_template('index.html', email=session.get('email', ''), tag=session.get('tag', ''))
    email = request.form['email']
    tag = request.form['tag']
    session['email'] = email
    session['tag'] = tag
    get_food.apply_async(args=[email, tag])
    flash('Sometimes this part takes a while. We\'ll send it all over to {0} when it\'s ready. Thanks for being patient!'.format(email))
    return render_template('index.html', email=session.get('email', ''), tag=session.get('tag', ''))

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>from apscheduler.scheduler import Scheduler
import time
import sys
import os
sys.path.insert(1, '/home/fabian/cos/scrapi/')
import requests

def main():
    config = {
        'apscheduler.jobstores.file.class': 'apscheduler.jobstores.shelve_store:ShelveJobStore',
        'apscheduler.jobstores.file.path': '/tmp/dbfile'
    }
    sched = Scheduler(config)

    sched.add_cron_job(plos, day_of_week='mon-fri', hour=23, minute=59)
    sched.start()

    print('Press Ctrl+{0} to exit'.format('C'))

    try:
        # This is here to simulate application activity (which keeps the main thread alive).
        while True:
            time.sleep(2)
    except (KeyboardInterrupt, SystemExit):
        sched.shutdown()  # Not strictly necessary if daemonic mode is enabled but should be done if possibleisched.start()

def plos():
    r = requests.post("http://localhost:1338/consume")
    print r
    for dirname, dirnames, filenames in os.walk('../archive/PLoS/'):
        # print path to all filenames.
        for filename in filenames:
            if 'raw' in filename:
                with open(os.path.join(dirname, filename), 'r') as f:
                    payload = {'doc': f.read()}
                    requests.post('http://localhost:1338/process', params=payload)  # TODO

if __name__ == '__main__':
    plos()
<commit_msg>added notes on what to do<commit_after>from apscheduler.scheduler import Scheduler
import time
import sys
import os
sys.path.insert(1, '/home/fabian/cos/scrapi/')
import requests

def main():
    config = {
        'apscheduler.jobstores.file.class': 'apscheduler.jobstores.shelve_store:ShelveJobStore',
        'apscheduler.jobstores.file.path': '/tmp/dbfile'
    }
    sched = Scheduler(config)

    sched.add_cron_job(plos, day_of_week='mon-fri', hour=23, minute=59)
    sched.start()

    print('Press Ctrl+{0} to exit'.format('C'))

    try:
        # This is here to simulate application activity (which keeps the main thread alive).
        while True:
            time.sleep(2)
    except (KeyboardInterrupt, SystemExit):
        sched.shutdown()  # Not strictly necessary if daemonic mode is enabled but should be done if possibleisched.start()

def plos():
    r = requests.post("http://localhost:1338/consume")
    print r
    for dirname, dirnames, filenames in os.walk('../archive/PLoS/'):
        # print path to all filenames.
        for filename in filenames:
            if 'raw' in filename:
                with open(os.path.join(dirname, filename), 'r') as f:
                    payload = {'doc': f.read()}
                    requests.post('http://localhost:1338/process', params=payload)  # TODO

def add_scraper():
    pass

if __name__ == '__main__':
    plos()

# programatically add a new scraper - generalizing the plos function
# manifest file loading System
# how to store and load various scrapers
# on some schedule, send requests to scrapers to consume info and parse it
# when you add a scraper, the manifest.json file defines where you live and where you want us to run
# create a cron job based on that info in the manifest.json file

<|endoftext|>


<commit_before>"""
:mod:`zsl.application.error_handler`
---------------------------------------------------

This module does the error handling. It allows users to register
an error handler for a given exception type. It also provides default
error handlers.
"""
from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import *
from functools import wraps
import http.client
import logging
import traceback
from typing import List

from flask import request

from zsl import inject
from zsl.db.model.app_model import AppModel
from zsl.errors import ErrorConfiguration, ErrorHandler, ErrorProcessor
from zsl.interface.task import ModelConversionError, ForbiddenError
from zsl.router.task import RoutingError
from zsl.task.job_context import JobContext, StatusCodeResponder, WebJobContext, add_responder
from zsl.task.task_decorator import json_output
from zsl.utils.documentation import documentation_link
from zsl.utils.http import get_http_status_code_value

class ErrorResponse(AppModel):
    def __init__(self, code, message):
        super(ErrorResponse, self).__init__({})
        self.code = code
        self.message = message

def register(e):
    # type: (ErrorHandler|ErrorProcessor)->None
    if isinstance(e, ErrorHandler):
        _error_handlers.append(e)
    if isinstance(e, ErrorProcessor):
        _error_processors.append(e)

class DefaultErrorHandler(ErrorHandler):
    ERROR_CODE = "UNKNOWN_ERROR"
    ERROR_MESSAGE = "An error occurred!"

    def can_handle(self, e):
        return True

    @json_output
    def handle(self, ex):
        logger = logging.getLogger(__name__)
        logger.error(str(ex) + "\n" + traceback.format_exc())
        logger.error("Request:\n{0}\n{1}\n".format(request.headers,
                                                   request.data))
        link = documentation_link('error_handling')
        logger.info("Provide your own error handler so that "
                    "a better error is produced, check {0}.".format(link))

        add_responder(StatusCodeResponder(http.client.INTERNAL_SERVER_ERROR))
        return ErrorResponse(self.ERROR_CODE, self.ERROR_MESSAGE)

class RoutingErrorHandler(ErrorHandler):
    ERROR_CODE = 'NOT_FOUND'

    def can_handle(self, e):
        return isinstance(e, RoutingError)

    @json_output
    def handle(self, ie):
        logging.error(str(ie) + "\n" + traceback.format_exc())
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.NOT_FOUND)))
        return ErrorResponse(self.ERROR_CODE, str(ie))

class ModelConversionErrorHandler(ErrorHandler):
    ERROR_CODE = 'INVALID_REQUEST'

    def can_handle(self, e):
        return isinstance(e, ModelConversionError)

    @json_output
    def handle(self, e):
        logging.error(str(e) + "\n" + traceback.format_exc())
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.UNPROCESSABLE_ENTITY)))
        return ErrorResponse(self.ERROR_CODE, str(e))

class ForbiddenErrorHandler(ErrorHandler):
    ERROR_CODE = 'FORBIDDEN'

    def can_handle(self, e):
        return isinstance(e, ForbiddenError)

    @json_output
    def handle(self, e):
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.FORBIDDEN)))
        return ErrorResponse(self.ERROR_CODE, str(e))

_DEFAULT_ERROR_HANDLER = DefaultErrorHandler()
_ROUTING_ERROR_HANDLER = RoutingErrorHandler()
_FORBIDDEN_ERROR_HANDLER = ForbiddenErrorHandler()
_MODEL_CONVERSION_ERROR_HANDLER = ModelConversionErrorHandler()
_error_handlers = [_MODEL_CONVERSION_ERROR_HANDLER, _FORBIDDEN_ERROR_HANDLER, _ROUTING_ERROR_HANDLER]  # type: List[ErrorHandler]
_error_processors = []

def error_handler(f):
    """
    Default error handler.
     - On server side error shows a message
       'An error occurred!' and returns 500 status code.
     - Also serves well in the case when the resource/task/method
       is not found - returns 404 status code.
    """

    @wraps(f)
    def error_handling_function(*args, **kwargs):
        @inject(error_config=ErrorConfiguration)
        def get_error_configuration(error_config):
            # type:(ErrorConfiguration)->ErrorConfiguration
            return error_config

        def should_skip_handling():
            use_flask_handler = get_error_configuration().use_flask_handler
            is_web_request = isinstance(JobContext.get_current_context(), WebJobContext)
            return use_flask_handler and is_web_request

        try:
            return f(*args, **kwargs)
        except Exception as ex:
            if should_skip_handling():
                raise

            for ep in _error_processors:
                ep.handle(ex)

            for eh in _error_handlers:
                if eh.can_handle(ex):
                    return eh.handle(ex)

            return _DEFAULT_ERROR_HANDLER.handle(ex)

    return error_handling_function
<commit_msg>fixing too long line<commit_after>"""
:mod:`zsl.application.error_handler`
---------------------------------------------------

This module does the error handling. It allows users to register
an error handler for a given exception type. It also provides default
error handlers.
"""
from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import *
from functools import wraps
import http.client
import logging
import traceback
from typing import List

from flask import request

from zsl import inject
from zsl.db.model.app_model import AppModel
from zsl.errors import ErrorConfiguration, ErrorHandler, ErrorProcessor
from zsl.interface.task import ModelConversionError, ForbiddenError
from zsl.router.task import RoutingError
from zsl.task.job_context import JobContext, StatusCodeResponder, WebJobContext, add_responder
from zsl.task.task_decorator import json_output
from zsl.utils.documentation import documentation_link
from zsl.utils.http import get_http_status_code_value

class ErrorResponse(AppModel):
    def __init__(self, code, message):
        super(ErrorResponse, self).__init__({})
        self.code = code
        self.message = message

def register(e):
    # type: (ErrorHandler|ErrorProcessor)->None
    if isinstance(e, ErrorHandler):
        _error_handlers.append(e)
    if isinstance(e, ErrorProcessor):
        _error_processors.append(e)

class DefaultErrorHandler(ErrorHandler):
    ERROR_CODE = "UNKNOWN_ERROR"
    ERROR_MESSAGE = "An error occurred!"

    def can_handle(self, e):
        return True

    @json_output
    def handle(self, ex):
        logger = logging.getLogger(__name__)
        logger.error(str(ex) + "\n" + traceback.format_exc())
        logger.error("Request:\n{0}\n{1}\n".format(request.headers,
                                                   request.data))
        link = documentation_link('error_handling')
        logger.info("Provide your own error handler so that "
                    "a better error is produced, check {0}.".format(link))

        add_responder(StatusCodeResponder(http.client.INTERNAL_SERVER_ERROR))
        return ErrorResponse(self.ERROR_CODE, self.ERROR_MESSAGE)

class RoutingErrorHandler(ErrorHandler):
    ERROR_CODE = 'NOT_FOUND'

    def can_handle(self, e):
        return isinstance(e, RoutingError)

    @json_output
    def handle(self, ie):
        logging.error(str(ie) + "\n" + traceback.format_exc())
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.NOT_FOUND)))
        return ErrorResponse(self.ERROR_CODE, str(ie))

class ModelConversionErrorHandler(ErrorHandler):
    ERROR_CODE = 'INVALID_REQUEST'

    def can_handle(self, e):
        return isinstance(e, ModelConversionError)

    @json_output
    def handle(self, e):
        logging.error(str(e) + "\n" + traceback.format_exc())
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.UNPROCESSABLE_ENTITY)))
        return ErrorResponse(self.ERROR_CODE, str(e))

class ForbiddenErrorHandler(ErrorHandler):
    ERROR_CODE = 'FORBIDDEN'

    def can_handle(self, e):
        return isinstance(e, ForbiddenError)

    @json_output
    def handle(self, e):
        add_responder(StatusCodeResponder(get_http_status_code_value(http.client.FORBIDDEN)))
        return ErrorResponse(self.ERROR_CODE, str(e))

_DEFAULT_ERROR_HANDLER = DefaultErrorHandler()
_ROUTING_ERROR_HANDLER = RoutingErrorHandler()
_FORBIDDEN_ERROR_HANDLER = ForbiddenErrorHandler()
_MODEL_CONVERSION_ERROR_HANDLER = ModelConversionErrorHandler()
_error_handlers = [_MODEL_CONVERSION_ERROR_HANDLER, _FORBIDDEN_ERROR_HANDLER,
                   _ROUTING_ERROR_HANDLER]  # type: List[ErrorHandler]
_error_processors = []

def error_handler(f):
    """
    Default error handler.
     - On server side error shows a message
       'An error occurred!' and returns 500 status code.
     - Also serves well in the case when the resource/task/method
       is not found - returns 404 status code.
    """

    @wraps(f)
    def error_handling_function(*args, **kwargs):
        @inject(error_config=ErrorConfiguration)
        def get_error_configuration(error_config):
            # type:(ErrorConfiguration)->ErrorConfiguration
            return error_config

        def should_skip_handling():
            use_flask_handler = get_error_configuration().use_flask_handler
            is_web_request = isinstance(JobContext.get_current_context(), WebJobContext)
            return use_flask_handler and is_web_request

        try:
            return f(*args, **kwargs)
        except Exception as ex:
            if should_skip_handling():
                raise

            for ep in _error_processors:
                ep.handle(ex)

            for eh in _error_handlers:
                if eh.can_handle(ex):
                    return eh.handle(ex)

            return _DEFAULT_ERROR_HANDLER.handle(ex)

    return error_handling_function
<|endoftext|>


<commit_before>#!flask/bin/python
from flask import Flask, jsonify
from error_messages import get_error_message
import t_rex_stateless as Trex
import thread
from cors_decorator import crossdomain

app = Flask(__name__)

config = {
    'api_version': 0.1,
    'is_running': False
}

def start_traffic(threadName, delay):

    print threadName, delay

    traffic_config = {
        'duration': 20,
        'rate': '10000pps',
        'warmup_time': 0,
        'async_start': False
    }

    traffic_options = {'p1_src_start_ip': '10.10.10.2', 'p2_src_start_ip': '20.20.20.2', 'p2_dst_start_ip': '10.10.10.2',
     'p2_src_end_ip': '20.20.20.254', 'p1_src_end_ip': '10.10.10.254', 'p1_dst_start_ip': '20.20.20.2'}

    pkt_a, pkt_b = Trex.create_packets(traffic_options, 1500)

    Trex.simple_burst(
        pkt_a, pkt_b, traffic_config['duration'], traffic_config['rate'], traffic_config['warmup_time'], traffic_config['async_start']
    )

    config['is_running'] = False

    return 0

# Check server status
@app.route('/')
@crossdomain(origin='*')
def index():
    return jsonify({
        'status': 'ok',
        'result': 'ok'
    })

# Get API version
@app.route('/api_version', methods=['GET'])
@crossdomain(origin='*')
def api_version():
    return jsonify({
        'status': 'ok',
        'result': config['api_version']
    })

# Start generating traffic
@app.route('/start', methods=['POST'])
@crossdomain(origin='*')
def start_trex(args):

    print

    if not config['is_running']:
        config['is_running']= True
        thread.start_new_thread(start_traffic, ("Thread-1", 2,))
    else:
        stop_trex()
        start_trex(args)

    return jsonify({
        'status': 'ok',
        'result': 'start'
    })

# Stop sending traffic
@app.route('/stop', methods=['POST'])
@crossdomain(origin='*')
def stop_trex():
    config['is_running'] = False
    Trex.stop_client()
    return jsonify({
        'status': 'ok',
        'result': 'stop'
    })

# Get TRex status
@app.route('/get_status', methods=['GET'])
@crossdomain(origin='*')
def get_status():
    return jsonify({
        'status': 'ok',
        'result': "running" if config['is_running'] else "stopped"
    })

# Not implemented methods
@app.errorhandler(404)
def not_implemented(e):
    return jsonify({
        'status': 'error',
        'result': get_error_message('not_implemented')
    }), 404
<commit_msg>New format of JSON response<commit_after>#!flask/bin/python
from flask import Flask, jsonify
from error_messages import get_error_message
import t_rex_stateless as Trex
import thread
from cors_decorator import crossdomain

app = Flask(__name__)

config = {
    'api_version': 0.1,
    'is_running': False
}

def responsify(status, result):
    return jsonify({
        'output': {
            'status': status,
            'result': result
        }
    })

def start_traffic(threadName, delay):

    print threadName, delay

    traffic_config = {
        'duration': 20,
        'rate': '10000pps',
        'warmup_time': 0,
        'async_start': False
    }

    traffic_options = {'p1_src_start_ip': '10.10.10.2', 'p2_src_start_ip': '20.20.20.2', 'p2_dst_start_ip': '10.10.10.2',
     'p2_src_end_ip': '20.20.20.254', 'p1_src_end_ip': '10.10.10.254', 'p1_dst_start_ip': '20.20.20.2'}

    pkt_a, pkt_b = Trex.create_packets(traffic_options, 1500)

    Trex.simple_burst(
        pkt_a, pkt_b, traffic_config['duration'], traffic_config['rate'], traffic_config['warmup_time'], traffic_config['async_start']
    )

    config['is_running'] = False

    return 0

# Check server status
@app.route('/')
@crossdomain(origin='*')
def index():
    return responsify('ok', 'ok')

# Get API version
@app.route('/api_version', methods=['GET'])
@crossdomain(origin='*')
def api_version():
    return responsify('ok', config['api_version'])

# Start generating traffic
@app.route('/start', methods=['POST'])
@crossdomain(origin='*')
def start_trex(args):

    print

    if not config['is_running']:
        config['is_running']= True
        thread.start_new_thread(start_traffic, ("Thread-1", 2,))
    else:
        stop_trex()
        start_trex(args)
    return responsify('ok', 'start')

# Stop sending traffic
@app.route('/stop', methods=['POST'])
@crossdomain(origin='*')
def stop_trex():
    config['is_running'] = False
    Trex.stop_client()
    return responsify('ok', 'stop')

# Get TRex status
@app.route('/get_status', methods=['GET'])
@crossdomain(origin='*')
def get_status():
    return responsify('ok', "running" if config['is_running'] else "stopped")

# Not implemented methods
@app.errorhandler(404)
def not_implemented(e):
    return responsify('error', get_error_message('not_implemented')), 404
<|endoftext|>


<commit_before>#!/usr/bin/env python

from cgi import parse_qs
import signapp

def application(environ, start_response):
	## passing environ uwsgi PARAM
	try:
		request_body_size = int(environ.get('CONTENT_LENGTH', 0))
	except (ValueError):
		request_body_size = 0
	uri = environ['REQUEST_URI']
	request_body = environ['wsgi.input'].read(request_body_size)
	post = parse_qs(request_body)
	## Declare apps
	sign = signapp.Signapp()
	## Menu Logic
	url=sign.urlDecode16(uri[1:])
	if sign.getMenu(url[:3])=="key":
		data = url[4:]
		result = ''
		for a in sign.getAllSign(data):
			result=result+str(a)
		hbegin = sign.getHtmlBegin()
		hend = sign.getHtmlEnd()
		tokenuriparam = sign.tokenUri()
		hend = hend.replace("TOKENURIPARAM",sign.urlEncode16(tokenuriparam),1)
		respon = hbegin + result + hend
	elif sign.getMenu(url[:3])=="token":
		token = post.get('token', [''])[0]
		html = sign.getTokenData(token)
		email = sign.getJsonData('email',html)
		respon = email
	else:
		result = url
		hbegin = sign.getHtmlBegin()
		hend = sign.getHtmlEnd()
		tokenuriparam = sign.tokenUri()
		hend = hend.replace("TOKENURIPARAM",sign.urlEncode16(tokenuriparam),1)
		respon = hbegin + result + hend
	## Passing HTML to client
	start_response('200 OK', [('Content-Type', 'text/html'),('Content-Length', str(len(respon)))])
	return [respon]

<commit_msg>get data array fix from 4 to 3<commit_after>#!/usr/bin/env python

from cgi import parse_qs
import signapp

def application(environ, start_response):
	## passing environ uwsgi PARAM
	try:
		request_body_size = int(environ.get('CONTENT_LENGTH', 0))
	except (ValueError):
		request_body_size = 0
	uri = environ['REQUEST_URI']
	request_body = environ['wsgi.input'].read(request_body_size)
	post = parse_qs(request_body)
	## Declare apps
	sign = signapp.Signapp()
	## Menu Logic
	url=sign.urlDecode16(uri[1:])
	if sign.getMenu(url[:3])=="key":
		data = url[3:]
		result = ''
		for a in sign.getAllSign(data):
			result=result+str(a)
		hbegin = sign.getHtmlBegin()
		hend = sign.getHtmlEnd()
		tokenuriparam = sign.tokenUri()
		hend = hend.replace("TOKENURIPARAM",sign.urlEncode16(tokenuriparam),1)
		respon = hbegin + result + hend
	elif sign.getMenu(url[:3])=="token":
		token = post.get('token', [''])[0]
		html = sign.getTokenData(token)
		email = sign.getJsonData('email',html)
		respon = email
	else:
		result = url
		hbegin = sign.getHtmlBegin()
		hend = sign.getHtmlEnd()
		tokenuriparam = sign.tokenUri()
		hend = hend.replace("TOKENURIPARAM",sign.urlEncode16(tokenuriparam),1)
		respon = hbegin + result + hend
	## Passing HTML to client
	start_response('200 OK', [('Content-Type', 'text/html'),('Content-Length', str(len(respon)))])
	return [respon]

<|endoftext|>


<commit_before>##  Changemonger: An OpenStreetMap change analyzer
##  Copyright (C) 2012 Serge Wroclawki
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as
##  published by the Free Software Foundation, either version 3 of the
##  License, or (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import Flask, jsonify, request, render_template
app = Flask(__name__)
app.debug = True
from features import precision
import helpers
from elements import common_name, display_name
from werkzeug import ImmutableDict
import changemonger

class FlaskWithHamlish(Flask):
    jinja_options = ImmutableDict(
        extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_',
                    'hamlish_jinja.HamlishExtension'] )

app = FlaskWithHamlish(__name__)
app.jinja_env.hamlish_mode = 'indented' # if you want to set hamlish settings
app.jinja_env.globals.update(precision=precision)

@app.route('/')
def index():
    return render_template('index.haml')

@app.route('/node')
def display_node():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_node_or_404(id)
        features = changemonger.features(ele)
        return render_template('node_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('node.haml', title="Bad Node")

@app.route('/way')
def display_way():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_way_or_404(id)
        features = changemonger.features(ele)
        return render_template('way_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('way.haml', title="Bad Way")

@app.route('/relation')
def display_relation():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_relation_or_404(id)
        features = changemonger.features(ele)
        return render_template('relation_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('relation.haml', title="Bad Way")

@app.route('/changeset')
def display_changeset():
    if request.args.has_key('id'):
        id = request.args['id']
        cset = helpers.get_changeset_or_404(id)
        sentence = changemonger.changeset_sentence(cset)
        return render_template('changeset_details.haml',
                               changeset = cset,
                               sentence = sentence)
    else:
        return render_template('changeset.haml', title="Bad Changeset")

@app.route('/features')
def show_features():
    return render_template('features.haml',
                           features = changemonger.db._features)

@app.route('/dbstats')
def show_stats():
    return jsonify(features=len(changemonger.db._features),
                   categories=len(changemonger.db._categories))

@app.route('/api/features/node/<id>')
def api_node(id):
    ele = helpers.get_node_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/features/way/<id>')
def api_way(id):
    ele = helpers.get_way_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/features/relation/<id>')
def api_relation(id):
    ele = helpers.get_relation_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/changeset/<changesetid>')
def show_changeset(id):
    cset = helpers.get_changeset_or_404(id)
    sentence = changemonger.changeset_sentence(cset)
    return sentence

if __name__ == '__main__':
    app.debug = True
    app.run()
<commit_msg>Fixing bug in API<commit_after>##  Changemonger: An OpenStreetMap change analyzer
##  Copyright (C) 2012 Serge Wroclawki
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as
##  published by the Free Software Foundation, either version 3 of the
##  License, or (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import Flask, jsonify, request, render_template
app = Flask(__name__)
app.debug = True
from features import precision
import helpers
from elements import common_name, display_name
from werkzeug import ImmutableDict
import changemonger

class FlaskWithHamlish(Flask):
    jinja_options = ImmutableDict(
        extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_',
                    'hamlish_jinja.HamlishExtension'] )

app = FlaskWithHamlish(__name__)
app.jinja_env.hamlish_mode = 'indented' # if you want to set hamlish settings
app.jinja_env.globals.update(precision=precision)

@app.route('/')
def index():
    return render_template('index.haml')

@app.route('/node')
def display_node():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_node_or_404(id)
        features = changemonger.features(ele)
        return render_template('node_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('node.haml', title="Bad Node")

@app.route('/way')
def display_way():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_way_or_404(id)
        features = changemonger.features(ele)
        return render_template('way_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('way.haml', title="Bad Way")

@app.route('/relation')
def display_relation():
    if request.args.has_key('id'):
        id = request.args['id']
        ele = helpers.get_relation_or_404(id)
        features = changemonger.features(ele)
        return render_template('relation_details.haml',
                               name = display_name(ele, features[0]),
                               node = ele,
                               features = features)
    else:
        return render_template('relation.haml', title="Bad Way")

@app.route('/changeset')
def display_changeset():
    if request.args.has_key('id'):
        id = request.args['id']
        cset = helpers.get_changeset_or_404(id)
        sentence = changemonger.changeset_sentence(cset)
        return render_template('changeset_details.haml',
                               changeset = cset,
                               sentence = sentence)
    else:
        return render_template('changeset.haml', title="Bad Changeset")

@app.route('/features')
def show_features():
    return render_template('features.haml',
                           features = changemonger.db._features)

@app.route('/dbstats')
def show_stats():
    return jsonify(features=len(changemonger.db._features),
                   categories=len(changemonger.db._categories))

@app.route('/api/features/node/<id>')
def api_node(id):
    ele = helpers.get_node_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/features/way/<id>')
def api_way(id):
    ele = helpers.get_way_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/features/relation/<id>')
def api_relation(id):
    ele = helpers.get_relation_or_404(id)
    features = changemonger.features(ele)
    features_names = [f.name for f in features]
    return jsonify(cn=common_name(ele),
                   features=features_names)

@app.route('/api/changeset/<id>')
def show_changeset(id):
    cset = helpers.get_changeset_or_404(id)
    sentence = changemonger.changeset_sentence(cset)
    return sentence

if __name__ == '__main__':
    app.debug = True
    app.run()
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pipes

from oslo.config import cfg
from st2common.constants.action import LIBS_DIR as ACTION_LIBS_DIR

__all__ = [
    'get_packs_base_paths',
    'check_pack_directory_exists',
    'check_pack_content_directory_exists',
    'get_pack_base_path'
]

def get_packs_base_paths():
    """
    Return a list of base paths which are searched for integration packs.

    :rtype: ``list``
    """
    packs_base_paths = cfg.CONF.content.packs_base_paths or ''
    packs_base_paths = packs_base_paths.split(':')
    return packs_base_paths

def check_pack_directory_exists(pack):
    """
    Check if a provided pack exists in one of the pack paths.

    :param pack: Pack name.
    :type pack: ``str``

    :rtype: ``bool``
    """
    packs_base_paths = get_packs_base_paths()

    for base_dir in packs_base_paths:
        pack_path = os.path.join(base_dir, pack)
        if os.path.exists(pack_path):
            return True

    return False

def check_pack_content_directory_exists(pack, content_type):
    """
    Check if a provided pack exists in one of the pack paths.

    :param pack: Pack name.
    :type pack: ``str``

    :param content_type: Content type (actions, sensors, rules).
    :type content_type: ``str``

    :rtype: ``bool``
    """
    packs_base_paths = get_packs_base_paths()

    for base_dir in packs_base_paths:
        pack_content_pack = os.path.join(base_dir, pack, content_type)
        if os.path.exists(pack_content_pack):
            return True

    return False

def get_pack_base_path(pack_name):
    """
    Return full absolute base path to the content pack directory.

    :param pack_name: Content pack name.
    :type pack_name: ``str``

    :rtype: ``str``
    """
    if not pack_name:
        return None

    packs_base_path = get_packs_base_path()
    pack_base_path = os.path.join(packs_base_path, pipes.quote(pack_name))
    pack_base_path = os.path.abspath(pack_base_path)
    return pack_base_path

def get_entry_point_abs_path(pack=None, entry_point=None):
    """
    Return full absolute path of an action entry point in a pack.

    :param pack_name: Content pack name.
    :type pack_name: ``str``
    :param entry_point: Action entry point.
    :type entry_point: ``str``

    :rtype: ``str``
    """
    if entry_point is not None and len(entry_point) > 0:
        if os.path.isabs(entry_point):
            return entry_point
        return os.path.join(get_packs_base_path(),
                            pipes.quote(pack), 'actions', pipes.quote(entry_point))
    else:
        return None

def get_action_libs_abs_path(pack=None, entry_point=None):
    """
    Return full absolute path of libs for an action.

    :param pack_name: Content pack name.
    :type pack_name: ``str``
    :param entry_point: Action entry point.
    :type entry_point: ``str``

    :rtype: ``str``
    """
    entry_point_abs_path = get_entry_point_abs_path(pack=pack, entry_point=entry_point)
    if entry_point_abs_path is not None:
        return os.path.join(os.path.dirname(entry_point_abs_path), ACTION_LIBS_DIR)
    else:
        return None
<commit_msg>Update more affected functions.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pipes

from oslo.config import cfg
from st2common.constants.action import LIBS_DIR as ACTION_LIBS_DIR

__all__ = [
    'get_packs_base_paths',
    'get_pack_base_path',
    'check_pack_directory_exists',
    'check_pack_content_directory_exists'
]

def get_packs_base_paths():
    """
    Return a list of base paths which are searched for integration packs.

    :rtype: ``list``
    """
    packs_base_paths = cfg.CONF.content.packs_base_paths or ''
    packs_base_paths = packs_base_paths.split(':')
    return packs_base_paths

def check_pack_directory_exists(pack):
    """
    Check if a provided pack exists in one of the pack paths.

    :param pack: Pack name.
    :type pack: ``str``

    :rtype: ``bool``
    """
    packs_base_paths = get_packs_base_paths()

    for base_dir in packs_base_paths:
        pack_path = os.path.join(base_dir, pack)
        if os.path.exists(pack_path):
            return True

    return False

def check_pack_content_directory_exists(pack, content_type):
    """
    Check if a provided pack exists in one of the pack paths.

    :param pack: Pack name.
    :type pack: ``str``

    :param content_type: Content type (actions, sensors, rules).
    :type content_type: ``str``

    :rtype: ``bool``
    """
    packs_base_paths = get_packs_base_paths()

    for base_dir in packs_base_paths:
        pack_content_pack = os.path.join(base_dir, pack, content_type)
        if os.path.exists(pack_content_pack):
            return True

    return False

def get_pack_base_path(pack_name):
    """
    Return full absolute base path to the content pack directory.

    Note: This function looks for a pack in all the load paths and return path to the first pack
    which matched the provided name. If pack is not found None is returned.

    :param pack_name: Content pack name.
    :type pack_name: ``str``

    :rtype: ``str``
    """
    if not pack_name:
        return None

    packs_base_paths = get_packs_base_paths()
    for packs_base_path in packs_base_paths:
        pack_base_path = os.path.join(packs_base_path, pipes.quote(pack_name))
        pack_base_path = os.path.abspath(pack_base_path)

        if os.path.isdir(pack_base_path):
            return pack_base_path

    # Path with the provided name not found
    return None

def get_entry_point_abs_path(pack=None, entry_point=None):
    """
    Return full absolute path of an action entry point in a pack.

    :param pack_name: Content pack name.
    :type pack_name: ``str``
    :param entry_point: Action entry point.
    :type entry_point: ``str``

    :rtype: ``str``
    """
    if entry_point is not None and len(entry_point) > 0:
        if os.path.isabs(entry_point):
            return entry_point

        pack_base_path = get_pack_base_path(pack_name=pack)
        entry_point_abs_path = os.path.join(pack_base_path, 'actions', pipes.quote(entry_point))
        return entry_point_abs_path
    else:
        return None

def get_action_libs_abs_path(pack=None, entry_point=None):
    """
    Return full absolute path of libs for an action.

    :param pack_name: Content pack name.
    :type pack_name: ``str``
    :param entry_point: Action entry point.
    :type entry_point: ``str``

    :rtype: ``str``
    """
    entry_point_abs_path = get_entry_point_abs_path(pack=pack, entry_point=entry_point)
    if entry_point_abs_path is not None:
        return os.path.join(os.path.dirname(entry_point_abs_path), ACTION_LIBS_DIR)
    else:
        return None
<|endoftext|>


<commit_before>import os
from sqlalchemy.exc import IntegrityError
from flask_sqlalchemy import SQLAlchemy
from flask import Flask, render_template, request

import redis
from rq import Queue
import requests

from urllib.parse import urlparse
from indexer import Indexer

app = Flask(__name__)
app.config['DEBUG'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL')
app.config['REDIS_URL'] = os.environ.get('REDIS_URL', 'redis://localhost:6379')
db = SQLAlchemy(app)
redis_conn = redis.from_url(app.config['REDIS_URL'])
q = Queue(connection=redis_conn)

from models import *

def index_esri_server(server_id):
    app.logger.info('Indexing ESRI server %s', server_id)
    server = EsriServer.query.get(server_id)

    if not server:
        app.logger.error('ESRI server %s was not found', server_id)
        return

    server.status = 'importing'
    db.session.add(server)
    db.session.commit()

    resulting_status = 'errored'
    try:
        indexer = Indexer(app.logger)
        services = indexer.get_services(server.url)
        for service in services:
            service_details = indexer.get_service_details(service.get('url'))

            db_service = Service(
                server=server,
                name=service.get('name'),
                service_type=service.get('type'),
                service_data=service_details,
            )
            db.session.add(db_service)

            layers = service_details.get('layers', [])
            for layer in layers:
                db_layer = Layer(
                    service=db_service,
                    name=layer.get('name'),
                    layer_data=layer,
                )
                db.session.add(db_layer)
        resulting_status = 'imported'
    except requests.exceptions.RequestException:
        app.logger.exception('Problem indexing ESRI server %s', server_id)
    except ValueError:
        app.logger.exception('Problem indexing ESRI server %s', server_id)

    server.status = resulting_status
    server.job_id = None
    db.session.add(server)
    db.session.commit()

@app.route('/', methods=['GET', 'POST'])
def index():
    errors = []

    errored_url = None
    if request.method == 'POST':
        url = request.form['url']

        url_parts = urlparse(url)

        if url_parts.scheme not in ('http', 'https'):
            errors.append('That URL is not valid.')
            errored_url = url
        else:
            server = EsriServer(url=url)
            db.session.add(server)
            try:
                db.session.commit()

                job = q.enqueue_call(
                    func='app.index_esri_server',
                    args=(server.id,),
                    result_ttl=5000,
                )

                server.status = 'queued'
                server.job_id = job.get_id()
                db.session.add(server)
                db.session.commit()
            except IntegrityError:
                errors.append('That URL has already been added.')

    servers = EsriServer.query.paginate(page=int(request.args.get('page', 1)))

    return render_template('index.html', servers=servers, errors=errors, errored_url=errored_url)

@app.route('/servers/<int:server_id>', methods=['GET'])
def show_server(server_id):
    server = EsriServer.query.get_or_404(server_id)

    return render_template('show_server.html', server=server)

@app.route('/servers/<int:server_id>/services/<int:service_id>', methods=['GET'])
def show_service(server_id, service_id):
    server = EsriServer.query.get_or_404(server_id)
    service = Service.query.get_or_404(service_id)

    return render_template('show_service.html', server=server, service=service)

@app.route('/search', methods=['GET'])
def search():
    results = Layer.query \
        .filter(Layer.name.ilike('%{}%'.format(request.args.get('q'))))

    which_server = request.args.get('server_id')
    if which_server and which_server.isdigit():
        results = results.join(Service).filter(Service.server_id == int(which_server))

    page = request.args.get('page', 1)
    if not isinstance(page, int) and not page.isdigit():
        page = 1

    results = results.paginate(page=page)

    return render_template('show_search.html', results=results)

if __name__ == '__main__':
    app.run()
<commit_msg>Set up flask-migrate<commit_after>import os
from sqlalchemy.exc import IntegrityError
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask import Flask, render_template, request

import redis
from rq import Queue
import requests

from urllib.parse import urlparse
from indexer import Indexer

app = Flask(__name__)
app.config['DEBUG'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL')
app.config['REDIS_URL'] = os.environ.get('REDIS_URL', 'redis://localhost:6379')
db = SQLAlchemy(app)
migrate = Migrate(app, db)
redis_conn = redis.from_url(app.config['REDIS_URL'])
q = Queue(connection=redis_conn)

from models import *

def index_esri_server(server_id):
    app.logger.info('Indexing ESRI server %s', server_id)
    server = EsriServer.query.get(server_id)

    if not server:
        app.logger.error('ESRI server %s was not found', server_id)
        return

    server.status = 'importing'
    db.session.add(server)
    db.session.commit()

    resulting_status = 'errored'
    try:
        indexer = Indexer(app.logger)
        services = indexer.get_services(server.url)
        for service in services:
            service_details = indexer.get_service_details(service.get('url'))

            db_service = Service(
                server=server,
                name=service.get('name'),
                service_type=service.get('type'),
                service_data=service_details,
            )
            db.session.add(db_service)

            layers = service_details.get('layers', [])
            for layer in layers:
                db_layer = Layer(
                    service=db_service,
                    name=layer.get('name'),
                    layer_data=layer,
                )
                db.session.add(db_layer)
        resulting_status = 'imported'
    except requests.exceptions.RequestException:
        app.logger.exception('Problem indexing ESRI server %s', server_id)
    except ValueError:
        app.logger.exception('Problem indexing ESRI server %s', server_id)

    server.status = resulting_status
    server.job_id = None
    db.session.add(server)
    db.session.commit()

@app.route('/', methods=['GET', 'POST'])
def index():
    errors = []

    errored_url = None
    if request.method == 'POST':
        url = request.form['url']

        url_parts = urlparse(url)

        if url_parts.scheme not in ('http', 'https'):
            errors.append('That URL is not valid.')
            errored_url = url
        else:
            server = EsriServer(url=url)
            db.session.add(server)
            try:
                db.session.commit()

                job = q.enqueue_call(
                    func='app.index_esri_server',
                    args=(server.id,),
                    result_ttl=5000,
                )

                server.status = 'queued'
                server.job_id = job.get_id()
                db.session.add(server)
                db.session.commit()
            except IntegrityError:
                errors.append('That URL has already been added.')

    servers = EsriServer.query.paginate(page=int(request.args.get('page', 1)))

    return render_template('index.html', servers=servers, errors=errors, errored_url=errored_url)

@app.route('/servers/<int:server_id>', methods=['GET'])
def show_server(server_id):
    server = EsriServer.query.get_or_404(server_id)

    return render_template('show_server.html', server=server)

@app.route('/servers/<int:server_id>/services/<int:service_id>', methods=['GET'])
def show_service(server_id, service_id):
    server = EsriServer.query.get_or_404(server_id)
    service = Service.query.get_or_404(service_id)

    return render_template('show_service.html', server=server, service=service)

@app.route('/search', methods=['GET'])
def search():
    results = Layer.query \
        .filter(Layer.name.ilike('%{}%'.format(request.args.get('q'))))

    which_server = request.args.get('server_id')
    if which_server and which_server.isdigit():
        results = results.join(Service).filter(Service.server_id == int(which_server))

    page = request.args.get('page', 1)
    if not isinstance(page, int) and not page.isdigit():
        page = 1

    results = results.paginate(page=page)

    return render_template('show_search.html', results=results)

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>import logging
import time

import yaml

from ... import exceptions
from .default import (
    Provider as AWSProvider,
    retry_on_throttling,
)

logger = logging.getLogger(__name__)

def get_change_set_name():
    """Return a valid Change Set Name.

    The name has to satisfy the following regex:
        [a-zA-Z][-a-zA-Z0-9]*

    And must be unique across all change sets.

    """
    return 'change-set-{}'.format(int(time.time()))

def requires_replacement(changeset):
    """Return the changes within the changeset that require replacement.

    Args:
        changeset (list): List of changes

    Returns:
        list: A list of changes that require replacement, if any.

    """
    return [r for r in changeset if r["ResourceChange"]["Replacement"] ==
            'True']

def get_raw_input(message):
    """ Just a wrapper for raw_input for testing purposes. """
    return raw_input(message)

def ask_for_approval(full_changeset=None, include_verbose=False):
    """Prompt the user for approval to execute a change set.

    Args:
        full_changeset (list, optional): A list of the full changeset that will
            be output if the user specifies verbose.
        include_verbose (bool, optional): Boolean for whether or not to include
            the verbose option

    """
    approval_options = ['y', 'n']
    if include_verbose:
        approval_options.append('v')

    approve = get_raw_input("Execute the above changes? [{}] ".format(
        '/'.join(approval_options)))

    if include_verbose and approve == "v":
        logger.info(
            "Full changeset:\n%s",
            yaml.safe_dump(full_changeset),
        )
        return ask_for_approval()
    elif approve != "y":
        raise exceptions.CancelExecution

def output_summary(fqn, action, changeset, replacements_only=False):
    """Log a summary of the changeset.

    Args:
        fqn (string): fully qualified name of the stack
        action (string): action to include in the log message
        changeset (list): AWS changeset
        replacements_only (bool, optional): boolean for whether or not we only
            want to list replacements

    """
    replacements = []
    changes = []
    for change in changeset:
        resource = change['ResourceChange']
        replacement = resource.get('Replacement') == 'True'
        summary = '- %s %s (%s)' % (
            resource['Action'],
            resource['LogicalResourceId'],
            resource['ResourceType'],
        )
        if replacement:
            replacements.append(summary)
        else:
            changes.append(summary)

    summary = ''
    if replacements:
        if not replacements_only:
            summary += 'Replacements:\n'
        summary += '\n'.join(replacements)
    if changes:
        if summary:
            summary += '\n'
        summary += 'Changes:\n%s' % ('\n'.join(changes))
    logger.info('%s %s:\n%s', fqn, action, summary)

def wait_till_change_set_complete(cfn_client, change_set_id, try_count=25,
                                  sleep_time=.5, max_sleep=3):
    """ Checks state of a changeset, returning when it is in a complete state.

    Since changesets can take a little bit of time to get into a complete
    state, we need to poll it until it does so. This will try to get the
    state `try_count` times, waiting `sleep_time` * 2 seconds between each try
    up to the `max_sleep` number of seconds. If, after that time, the changeset
    is not in a complete state it fails. These default settings will wait a
    little over one minute.

    Args:
        cfn_client (:class:`botocore.client.CloudFormation`): Used to query
            cloudformation.
        change_set_id (str): The unique changeset id to wait for.
        try_count (int): Number of times to try the call.
        sleep_time (int): Time to sleep between attempts.
        max_sleep (int): Max time to sleep during backoff

    Return:
        dict: The response from cloudformation for the describe_change_set
            call.
    """
    complete = False
    response = None
    for i in range(try_count):
        response = retry_on_throttling(
            cfn_client.describe_change_set,
            kwargs={
                'ChangeSetName': change_set_id,
            },
        )
        complete = response["Status"] in ("FAILED", "CREATE_COMPLETE")
        if complete:
            break
        time.sleep(sleep_time)

        # exponential backoff with max
        sleep_time = min(sleep_time * 2, max_sleep)
    if not complete:
        raise exceptions.ChangesetDidNotStabilize(change_set_id)
    return response

def create_change_set(cfn_client, fqn, template_url, parameters, tags,
                      replacements_only=False):
    logger.debug("Attempting to create change set for stack: %s.", fqn)
    response = retry_on_throttling(
        cfn_client.create_change_set,
        kwargs={
            'StackName': fqn,
            'TemplateURL': template_url,
            'Parameters': parameters,
            'Tags': tags,
            'Capabilities': ["CAPABILITY_NAMED_IAM"],
            'ChangeSetName': get_change_set_name(),
        },
    )
    change_set_id = response["Id"]
    response = wait_till_change_set_complete(
        cfn_client, change_set_id
    )
    status = response["Status"]
    if status == "FAILED":
        status_reason = response["StatusReason"]
        if "didn't contain changes" in response["StatusReason"]:
            logger.debug(
                "Stack %s did not change, not updating.",
                fqn,
            )
            raise exceptions.StackDidNotChange
        raise exceptions.UnhandledChangeSetStatus(
            fqn, change_set_id, status, status_reason
        )

    execution_status = response["ExecutionStatus"]
    if execution_status != "AVAILABLE":
        raise exceptions.UnableToExecuteChangeSet(fqn,
                                                  change_set_id,
                                                  execution_status)

    changes = response["Changes"]
    return changes, change_set_id

class Provider(AWSProvider):
    """AWS Cloudformation Change Set Provider"""

    def __init__(self, region, replacements_only=False, *args, **kwargs):
        self.replacements_only = replacements_only
        super(Provider, self).__init__(region=region, *args, **kwargs)

    def update_stack(self, fqn, template_url, parameters, tags, diff=False,
                     **kwargs):
        changes, change_set_id = create_change_set(self.cloudformation, fqn,
                                                   template_url, parameters,
                                                   tags, **kwargs)

        action = "replacements" if self.replacements_only else "changes"
        full_changeset = changes
        if self.replacements_only:
            changes = requires_replacement(changes)

        if changes:
            output_summary(fqn, action, changes,
                           replacements_only=self.replacements_only)
            if not diff:
                ask_for_approval(
                    full_changeset=full_changeset,
                    include_verbose=True,
                )

        if not diff:
            retry_on_throttling(
                self.cloudformation.execute_change_set,
                kwargs={
                    'ChangeSetName': change_set_id,
                },
            )
        else:
            retry_on_throttling(
                self.cloudformation.delete_change_set,
                kwargs={
                    'ChangeSetName': change_set_id,
                },
            )

        return True
<commit_msg>Remove trailing whitespace<commit_after>import logging
import time

import yaml

from ... import exceptions
from .default import (
    Provider as AWSProvider,
    retry_on_throttling,
)

logger = logging.getLogger(__name__)

def get_change_set_name():
    """Return a valid Change Set Name.

    The name has to satisfy the following regex:
        [a-zA-Z][-a-zA-Z0-9]*

    And must be unique across all change sets.

    """
    return 'change-set-{}'.format(int(time.time()))

def requires_replacement(changeset):
    """Return the changes within the changeset that require replacement.

    Args:
        changeset (list): List of changes

    Returns:
        list: A list of changes that require replacement, if any.

    """
    return [r for r in changeset if r["ResourceChange"]["Replacement"] ==
            'True']

def get_raw_input(message):
    """ Just a wrapper for raw_input for testing purposes. """
    return raw_input(message)

def ask_for_approval(full_changeset=None, include_verbose=False):
    """Prompt the user for approval to execute a change set.

    Args:
        full_changeset (list, optional): A list of the full changeset that will
            be output if the user specifies verbose.
        include_verbose (bool, optional): Boolean for whether or not to include
            the verbose option

    """
    approval_options = ['y', 'n']
    if include_verbose:
        approval_options.append('v')

    approve = get_raw_input("Execute the above changes? [{}] ".format(
        '/'.join(approval_options)))

    if include_verbose and approve == "v":
        logger.info(
            "Full changeset:\n%s",
            yaml.safe_dump(full_changeset),
        )
        return ask_for_approval()
    elif approve != "y":
        raise exceptions.CancelExecution

def output_summary(fqn, action, changeset, replacements_only=False):
    """Log a summary of the changeset.

    Args:
        fqn (string): fully qualified name of the stack
        action (string): action to include in the log message
        changeset (list): AWS changeset
        replacements_only (bool, optional): boolean for whether or not we only
            want to list replacements

    """
    replacements = []
    changes = []
    for change in changeset:
        resource = change['ResourceChange']
        replacement = resource.get('Replacement') == 'True'
        summary = '- %s %s (%s)' % (
            resource['Action'],
            resource['LogicalResourceId'],
            resource['ResourceType'],
        )
        if replacement:
            replacements.append(summary)
        else:
            changes.append(summary)

    summary = ''
    if replacements:
        if not replacements_only:
            summary += 'Replacements:\n'
        summary += '\n'.join(replacements)
    if changes:
        if summary:
            summary += '\n'
        summary += 'Changes:\n%s' % ('\n'.join(changes))
    logger.info('%s %s:\n%s', fqn, action, summary)

def wait_till_change_set_complete(cfn_client, change_set_id, try_count=25,
                                  sleep_time=.5, max_sleep=3):
    """ Checks state of a changeset, returning when it is in a complete state.

    Since changesets can take a little bit of time to get into a complete
    state, we need to poll it until it does so. This will try to get the
    state `try_count` times, waiting `sleep_time` * 2 seconds between each try
    up to the `max_sleep` number of seconds. If, after that time, the changeset
    is not in a complete state it fails. These default settings will wait a
    little over one minute.

    Args:
        cfn_client (:class:`botocore.client.CloudFormation`): Used to query
            cloudformation.
        change_set_id (str): The unique changeset id to wait for.
        try_count (int): Number of times to try the call.
        sleep_time (int): Time to sleep between attempts.
        max_sleep (int): Max time to sleep during backoff

    Return:
        dict: The response from cloudformation for the describe_change_set
            call.
    """
    complete = False
    response = None
    for i in range(try_count):
        response = retry_on_throttling(
            cfn_client.describe_change_set,
            kwargs={
                'ChangeSetName': change_set_id,
            },
        )
        complete = response["Status"] in ("FAILED", "CREATE_COMPLETE")
        if complete:
            break
        time.sleep(sleep_time)

        # exponential backoff with max
        sleep_time = min(sleep_time * 2, max_sleep)
    if not complete:
        raise exceptions.ChangesetDidNotStabilize(change_set_id)
    return response

def create_change_set(cfn_client, fqn, template_url, parameters, tags,
                      replacements_only=False):
    logger.debug("Attempting to create change set for stack: %s.", fqn)
    response = retry_on_throttling(
        cfn_client.create_change_set,
        kwargs={
            'StackName': fqn,
            'TemplateURL': template_url,
            'Parameters': parameters,
            'Tags': tags,
            'Capabilities': ["CAPABILITY_NAMED_IAM"],
            'ChangeSetName': get_change_set_name(),
        },
    )
    change_set_id = response["Id"]
    response = wait_till_change_set_complete(
        cfn_client, change_set_id
    )
    status = response["Status"]
    if status == "FAILED":
        status_reason = response["StatusReason"]
        if "didn't contain changes" in response["StatusReason"]:
            logger.debug(
                "Stack %s did not change, not updating.",
                fqn,
            )
            raise exceptions.StackDidNotChange
        raise exceptions.UnhandledChangeSetStatus(
            fqn, change_set_id, status, status_reason
        )

    execution_status = response["ExecutionStatus"]
    if execution_status != "AVAILABLE":
        raise exceptions.UnableToExecuteChangeSet(fqn,
                                                  change_set_id,
                                                  execution_status)

    changes = response["Changes"]
    return changes, change_set_id

class Provider(AWSProvider):
    """AWS Cloudformation Change Set Provider"""

    def __init__(self, region, replacements_only=False, *args, **kwargs):
        self.replacements_only = replacements_only
        super(Provider, self).__init__(region=region, *args, **kwargs)

    def update_stack(self, fqn, template_url, parameters, tags, diff=False,
                     **kwargs):
        changes, change_set_id = create_change_set(self.cloudformation, fqn,
                                                   template_url, parameters,
                                                   tags, **kwargs)

        action = "replacements" if self.replacements_only else "changes"
        full_changeset = changes
        if self.replacements_only:
            changes = requires_replacement(changes)

        if changes:
            output_summary(fqn, action, changes,
                           replacements_only=self.replacements_only)
            if not diff:
                ask_for_approval(
                    full_changeset=full_changeset,
                    include_verbose=True,
                )

        if not diff:
            retry_on_throttling(
                self.cloudformation.execute_change_set,
                kwargs={
                    'ChangeSetName': change_set_id,
                },
            )
        else:
            retry_on_throttling(
                self.cloudformation.delete_change_set,
                kwargs={
                    'ChangeSetName': change_set_id,
                },
            )

        return True
<|endoftext|>


<commit_before># Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import paddle.fluid as fluid
import paddle.fluid.core as core
import numpy as np
import unittest
import os
import sys
import math

def simple_fc_net():
    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
    hidden = img
    for _ in range(4):
        hidden = fluid.layers.fc(
            hidden,
            size=200,
            act='tanh',
            bias_attr=fluid.ParamAttr(
                initializer=fluid.initializer.Constant(value=1.0)))
    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
    loss = fluid.layers.cross_entropy(input=prediction, label=label)
    loss = fluid.layers.mean(loss)
    return loss

class ParallelExecutorTestingDuringTraining(unittest.TestCase):
    def check_network_convergence(self, use_cuda, build_strategy=None):
        os.environ['CPU_NUM'] = str(4)
        main = fluid.Program()
        startup = fluid.Program()
        with fluid.program_guard(main, startup):
            loss = simple_fc_net()
            test_program = main.clone(for_test=True)

            opt = fluid.optimizer.SGD(learning_rate=0.001)
            opt.minimize(loss)

            batch_size = 32
            image = np.random.normal(size=(batch_size, 784)).astype('float32')
            label = np.random.randint(0, 10, (batch_size, 1), dtype="int64")

            place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
            exe = fluid.Executor(place)
            exe.run(startup)
            feed_dict = {'image': image, 'label': label}

            train_exe = fluid.ParallelExecutor(
                use_cuda=use_cuda,
                loss_name=loss.name,
                main_program=main,
                build_strategy=build_strategy)

            test_exe = fluid.ParallelExecutor(
                use_cuda=use_cuda,
                main_program=test_program,
                share_vars_from=train_exe,
                build_strategy=build_strategy)

            for i in range(5):
                test_loss, = test_exe.run([loss.name], feed=feed_dict)

                train_loss, = train_exe.run([loss.name], feed=feed_dict)

                avg_test_loss_val = np.array(test_loss).mean()
                if math.isnan(float(avg_test_loss_val)):
                    sys.exit("got NaN loss, testing failed.")

                avg_train_loss_val = np.array(train_loss).mean()
                if math.isnan(float(avg_train_loss_val)):
                    sys.exit("got NaN loss, training failed.")

                self.assertTrue(
                    np.allclose(
                        train_loss, test_loss, atol=1e-8),
                    "Train loss: " + str(train_loss) + "\n Test loss:" +
                    str(test_loss))

    def test_parallel_testing(self):
        build_strategy = fluid.BuildStrategy()
        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce
        if core.is_compiled_with_cuda():
            self.check_network_convergence(
                use_cuda=True, build_strategy=build_strategy)
        self.check_network_convergence(
            use_cuda=False, build_strategy=build_strategy)

    def test_parallel_testing_with_new_strategy(self):
        build_strategy = fluid.BuildStrategy()
        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
        if core.is_compiled_with_cuda():
            self.check_network_convergence(
                use_cuda=True, build_strategy=build_strategy)
        self.check_network_convergence(
            use_cuda=False, build_strategy=build_strategy)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Loose atol in test_pe_test_wwhile_train<commit_after># Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import paddle.fluid as fluid
import paddle.fluid.core as core
import numpy as np
import unittest
import os
import sys
import math

def simple_fc_net():
    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
    hidden = img
    for _ in range(4):
        hidden = fluid.layers.fc(
            hidden,
            size=200,
            act='tanh',
            bias_attr=fluid.ParamAttr(
                initializer=fluid.initializer.Constant(value=1.0)))
    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
    loss = fluid.layers.cross_entropy(input=prediction, label=label)
    loss = fluid.layers.mean(loss)
    return loss

class ParallelExecutorTestingDuringTraining(unittest.TestCase):
    def check_network_convergence(self, use_cuda, build_strategy=None):
        os.environ['CPU_NUM'] = str(4)
        main = fluid.Program()
        startup = fluid.Program()
        with fluid.program_guard(main, startup):
            loss = simple_fc_net()
            test_program = main.clone(for_test=True)

            opt = fluid.optimizer.SGD(learning_rate=0.001)
            opt.minimize(loss)

            batch_size = 32
            image = np.random.normal(size=(batch_size, 784)).astype('float32')
            label = np.random.randint(0, 10, (batch_size, 1), dtype="int64")

            place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
            exe = fluid.Executor(place)
            exe.run(startup)
            feed_dict = {'image': image, 'label': label}

            train_exe = fluid.ParallelExecutor(
                use_cuda=use_cuda,
                loss_name=loss.name,
                main_program=main,
                build_strategy=build_strategy)

            test_exe = fluid.ParallelExecutor(
                use_cuda=use_cuda,
                main_program=test_program,
                share_vars_from=train_exe,
                build_strategy=build_strategy)

            for i in range(5):
                test_loss, = test_exe.run([loss.name], feed=feed_dict)

                train_loss, = train_exe.run([loss.name], feed=feed_dict)

                avg_test_loss_val = np.array(test_loss).mean()
                if math.isnan(float(avg_test_loss_val)):
                    sys.exit("got NaN loss, testing failed.")

                avg_train_loss_val = np.array(train_loss).mean()
                if math.isnan(float(avg_train_loss_val)):
                    sys.exit("got NaN loss, training failed.")

                self.assertTrue(
                    np.allclose(
                        train_loss, test_loss, atol=1e-2),
                    "Train loss: " + str(train_loss) + "\n Test loss:" +
                    str(test_loss))

    def test_parallel_testing(self):
        build_strategy = fluid.BuildStrategy()
        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce
        if core.is_compiled_with_cuda():
            self.check_network_convergence(
                use_cuda=True, build_strategy=build_strategy)
        self.check_network_convergence(
            use_cuda=False, build_strategy=build_strategy)

    def test_parallel_testing_with_new_strategy(self):
        build_strategy = fluid.BuildStrategy()
        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
        if core.is_compiled_with_cuda():
            self.check_network_convergence(
                use_cuda=True, build_strategy=build_strategy)
        self.check_network_convergence(
            use_cuda=False, build_strategy=build_strategy)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import urllib
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
from flask import Flask
from flask import request
from flask import make_response

# Flask app should start in global layout
app = Flask(__name__)
intent_name="string"
@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)
    print("Request:")
    print(json.dumps(req, indent=4))
    res = processRequest(req)
    res = json.dumps(res, indent=4)
    print("after json.dumps",res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def processRequest(req):
    if req.get("result").get("action") != "property_search":
        return {}
    global city_names
    global QR
    global intent_name
    intent_name=processIntentName(req)
    city_names=processlocation(req)
    property_type=processPropertyType(req)
    maximum_value=processMaximum(req)
    #baseurl = "https://aarz.pk/bot/index.php?city_name="+city_names+"&sector_name="+sector_names+"&minPrice="+maximum_value+"&type="+property_type+"&LatestProperties="+latest+"&UnitArea="+area_property+"&Unit="+unit_property+"&school="+school+"&airport="+airport+"&transport="+transport+"&security="+security+"&shopping_mall="+malls+"&fuel="+fuel
    #baseurl="https://www.aarz.pk/search/bot?postedBy=searchPage&view=&city_s="+city_names+"&price_min="+maximum_value+"&price_max=0estate_agent=&purpose=Sell&property_type="+property_type

    baseurl="https://www.aarz.pk/search/bot?postedBy=searchPage&view=&city_s="+city_names+"&type="+property_type+"&maxPrice="+maximum_value
    print("city:",city_names)
    print("url is:",baseurl)
    result = urllib.request.urlopen(baseurl).read()
    print('result of url:', result)
    data = json.loads(result)
    print('data:', data)
    res2=json_to_text(data)
    print('res2:',res2)
    return res2

def processIntentName(req):
    result = req.get("result")
    parameters = result.get("metadata")
    intent = parameters.get("intentName")
    return intent

def processlocation(req):
    result = req.get("result")
    parameters = result.get("parameters")
    cityNames = (parameters.get("location"))!=''?(parameters.get("location")):(city='')
    city= cityNames.get("city")
    print("city data:", city)
    print("city:", city)

    return city

def processMaximum(req):
    result = req.get("result")
    parameters = result.get("parameters")
    maximum = parameters.get("PriceRange")
    return maximum

def processMinimum(req):
    result = req.get("result")
    parameters = result.get("parameters")
    min_price = parameters.get("min_price")
    return min_price

def processPropertyType(req):
    result = req.get("result")
    parameters = result.get("parameters")
    propertyType = parameters.get("PropertyType")
    return propertyType

def processProjectName(req):
    result = req.get("result")
    parameters = result.get("parameters")
    project_name = parameters.get("ProjectName")
    return project_name

def json_to_text(data):
     i=0
     length=len(data)
     speech="Here are some properties with your choice:We have total of "+str(length)+"records of you interest."
     row_id=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_title=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_location=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_price=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_slug=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_number=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_image=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_city=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     while (i <length):
        row_id[i]=data[i]['property_id']
        row_title[i]=data[i]['title']
        row_location[i]=data[i]['address']
        row_price[i]=data[i]['price']
        row_slug[i]=data[i]['slug']
        row_number[i]=data[i]['number']
        row_image[i]=data[i]['image']
        row_city[i]=data[i]['city_name']
        speech_parts = "Here is record " + str(i) +":"+ row_title[i]+" in city "+row_city[i] + " price is "+ str(row_price[i])+ "For Info about this contact at number "+str(row_number[i])
        speech=speech+speech_parts
        i+=1
     print('speech',speech)
     return {
        "speech": speech,
        "displayText": speech,
        "data": {},
        "contextOut": [],
        "source": "apiai-onlinestore-shipping"
    }
if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<commit_msg>indention added<commit_after>#!/usr/bin/env python

import urllib
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
from flask import Flask
from flask import request
from flask import make_response

# Flask app should start in global layout
app = Flask(__name__)
intent_name="string"
@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)
    print("Request:")
    print(json.dumps(req, indent=4))
    res = processRequest(req)
    res = json.dumps(res, indent=4)
    print("after json.dumps",res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def processRequest(req):
    if req.get("result").get("action") != "property_search":
        return {}
    global city_names
    global QR
    global intent_name
    intent_name=processIntentName(req)
    city_names=processlocation(req)
    property_type=processPropertyType(req)
    maximum_value=processMaximum(req)
    #baseurl = "https://aarz.pk/bot/index.php?city_name="+city_names+"&sector_name="+sector_names+"&minPrice="+maximum_value+"&type="+property_type+"&LatestProperties="+latest+"&UnitArea="+area_property+"&Unit="+unit_property+"&school="+school+"&airport="+airport+"&transport="+transport+"&security="+security+"&shopping_mall="+malls+"&fuel="+fuel
    #baseurl="https://www.aarz.pk/search/bot?postedBy=searchPage&view=&city_s="+city_names+"&price_min="+maximum_value+"&price_max=0estate_agent=&purpose=Sell&property_type="+property_type

    baseurl="https://www.aarz.pk/search/bot?postedBy=searchPage&view=&city_s="+city_names+"&type="+property_type+"&maxPrice="+maximum_value
    print("city:",city_names)
    print("url is:",baseurl)
    result = urllib.request.urlopen(baseurl).read()
    print('result of url:', result)
    data = json.loads(result)
    print('data:', data)
    res2=json_to_text(data)
    print('res2:',res2)
    return res2

def processIntentName(req):
    result = req.get("result")
    parameters = result.get("metadata")
    intent = parameters.get("intentName")
    return intent

def processlocation(req):
    result = req.get("result")
    parameters = result.get("parameters")
    cityNames = (parameters.get("location"))!=''?(parameters.get("location")):(city='')
    city= cityNames.get("city")
    print("city data:", city)
    print("city:", city)

    return city

def processMaximum(req):
    result = req.get("result")
    parameters = result.get("parameters")
    maximum = parameters.get("PriceRange")
    return maximum

def processMinimum(req):
    result = req.get("result")
    parameters = result.get("parameters")
    min_price = parameters.get("min_price")
    return min_price

def processPropertyType(req):
    result = req.get("result")
    parameters = result.get("parameters")
    propertyType = parameters.get("PropertyType")
    return propertyType

def processProjectName(req):
    result = req.get("result")
    parameters = result.get("parameters")
    project_name = parameters.get("ProjectName")
    return project_name

def json_to_text(data):
     i=0
     length=len(data)
     speech="Here are some properties with your choice. We have total of "+str(length)+" records of you interest. "
     row_id=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_title=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_location=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_price=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_slug=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_number=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_image=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     row_city=['test','test1','test2','test3','test4','test5','test6','test7','test8','test9','test10']
     while (i <length):
        row_id[i]=data[i]['property_id']
        row_title[i]=data[i]['title']
        row_location[i]=data[i]['address']
        row_price[i]=data[i]['price']
        row_slug[i]=data[i]['slug']
        row_number[i]=data[i]['number']
        row_image[i]=data[i]['image']
        row_city[i]=data[i]['city_name']
        speech_parts = "Here is record " + str(i+1) +":"+ row_title[i]+" in city "+row_city[i] + " price is "+ str(row_price[i])+ ". For Info about this contact at number "+str(row_number[i])
        speech=speech+speech_parts
        i+=1
     print('speech',speech)
     return {
        "speech": speech,
        "displayText": speech,
        "data": {},
        "contextOut": [],
        "source": "apiai-onlinestore-shipping"
    }
if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<|endoftext|>