<commit_before>"""Module that generates report graphs."""
from bokeh.plotting import figure
from bokeh.embed import components
from bokeh.models import DatetimeTickFormatter, Legend
from bokeh.palettes import Plasma256, linear_palette
from flask_login import current_user
from .database import db
from .database import Transaction, Category, Account
from sqlalchemy.sql import func
from collections import OrderedDict
from numpy import pi

def graph(report_name):
    """Report graph."""
    if report_name == "Expenses by Category":
        graph = ExpensesByCategoryPieGraph()
    elif report_name == "Income by Category":
        graph = IncomeByCategoryPieGraph()
    elif report_name == "Cash Flow":
        graph = CashFlowLineGraph()
    elif report_name == "Account Balances":
        graph = AccountBalancesLineGraph()
    return graph.get_html()

class PieGraph():
    """Pie graph."""

    def __init__(self):
        """Initialise."""
        self.data = []

    def get_html(self):
        """Get HTML components."""
        if self.data:
            amounts = [0]
            labels = []
            for row in self.data:
                labels.append(row[0])
                amounts.append(row[1])
        else:
            labels = ["No Data"]
            amounts = [0, 100]
        total = sum(amounts)
        amounts = [value / total for value in amounts]
        running_totals = []
        running_totals.append(amounts[0])
        running_totals.append(amounts[1])
        for num, value in enumerate(amounts):
            if num == 0 or num == 1:
                continue
            running_totals.append(sum(amounts[0:num + 1]))
        start_angles = [2 * pi * value for value in running_totals[:-1]]
        end_angles = [2 * pi * value for value in running_totals[1:]]
        num_colors = len(labels)
        if num_colors <= 256:
            colors = linear_palette(Plasma256, num_colors)
        else:
            colors = linear_palette(Plasma256, 256) * int(num_colors / 256 + 1)
        pie_chart = figure(
            x_range=(-1, 1), y_range=(-1, 1), plot_width=800, plot_height=600,
            logo=None)
        pie_chart.xaxis.visible = False
        pie_chart.yaxis.visible = False

        legend_items = []
        for num, label in enumerate(labels):
            wedge = pie_chart.wedge(
                x=0, y=0, radius=0.75, start_angle=start_angles[num],
                end_angle=end_angles[num], color=colors[num])
            percent = ' ' + str(round(amounts[num+1] * 100, 1)) + '%'
            legend_items.append((label + percent, [wedge]))

        legend = Legend(items=legend_items, location=(0, 0))
        pie_chart.add_layout(legend, 'right')

        script, div = components(pie_chart)
        return script, div

class ExpensesByCategoryPieGraph(PieGraph):
    """Expenses by category pie graph."""

    def __init__(self):
        """Perform database query."""
        super().__init__()
        self.data = (
            db.session.query(Category.catname, func.sum(Transaction.amount))
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .filter(Category.cattype == 'Expense')
            .group_by(Category.catname)
            .order_by(func.sum(Transaction.amount))
            .all())

class IncomeByCategoryPieGraph(PieGraph):
    """Income by category pie graph."""

    def __init__(self):
        """Perform database query."""
        super().__init__()
        self.data = (
            db.session.query(Category.catname, func.sum(Transaction.amount))
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .filter(Category.cattype == 'Income')
            .group_by(Category.catname)
            .order_by(func.sum(Transaction.amount))
            .all())

class LineGraph():
    """
    Line graph.

    Expects data to be { item1: [[x-values], [y-values]],
                         item2: [[x-values], [y-values]],
                         ...                              }
    """

    def __init__(self):
        """Initialise."""
        self.data = OrderedDict()

    def get_html(self):
        """Get HTML components."""
        plot = figure(
            x_axis_type='datetime', x_axis_label='Date', y_axis_label='Amount',
            logo=None)

        DATE_TIME_FORMAT = {
            'days': ['%d/%m/%y'], 'months': ['%m/%Y']}
        plot.xaxis.formatter = DatetimeTickFormatter(**DATE_TIME_FORMAT)
        num_colors = len(self.data)
        if num_colors > 256:
            num_colors = 256
        colors = linear_palette(Plasma256, len(self.data))
        items = []
        if self.data:
            for num, label in enumerate(self.data.keys()):
                dates = self.data[label][0]
                amounts = self.data[label][1]
                num = num % num_colors  # Start colors again if all used
                line = plot.line(
                    dates, amounts, line_color=colors[num], line_width=2)
                items.append((label, [line]))
        plot.sizing_mode = 'scale_width'

        legend = Legend(items=items, location=(0, 0))

        plot.add_layout(legend, 'below')

        script, div = components(plot)
        return script, div

class AccountBalancesLineGraph(LineGraph):
    """Account balances line graph."""

    def __init__(self):
        """Perform database query and populate data structure."""
        super().__init__()
        accounts = (
            db.session.query(Account.accname, Account.accno)
            .filter(Account.id == current_user.id)
            .order_by(Account.accname)
            .all())
        for accname, accno in accounts:
            balance = (
                db.session.query(func.sum(Account.balance))
                .filter(Account.id == current_user.id)
                .filter(Account.accname == accname)
                .first()[0] / 100.0)
            transactions = (
                db.session.query(
                    Transaction.date, Transaction.amount, Category.cattype)
                .filter(Transaction.id == current_user.id)
                .filter(Transaction.accno == accno)
                .filter(Transaction.catno == Category.catno)
                .order_by(Transaction.date)
                .all())
            dates = []
            amounts = []
            for date, amount, cattype in transactions:
                if cattype == 'Expense' or cattype == 'Transfer Out':
                    balance -= amount / 100.0
                else:
                    balance += amount / 100.0
                dates.append(date)
                amounts.append(balance)
            self.data[accname] = [dates, amounts]

class CashFlowLineGraph(LineGraph):
    """Cash flow line graph."""

    def __init__(self):
        """Perform database query and populate data structure."""
        super().__init__()
        balance = (
            db.session.query(func.sum(Account.balance))
            .filter(Account.id == current_user.id)
            .first()[0] / 100.0)
        transactions = (
            db.session.query(
                Transaction.date, Transaction.amount, Category.cattype)
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .order_by(Transaction.date)
            .all())
        dates = []
        amounts = []
        for date, amount, cattype in transactions:
            if cattype == 'Expense' or cattype == 'Transfer Out':
                balance -= amount / 100.0
            else:
                balance += amount / 100.0
            dates.append(date)
            amounts.append(balance)
        self.data['Net Total'] = [dates, amounts]
<commit_msg>Make minor cosmetic changes to pie graphs<commit_after>"""Module that generates report graphs."""
from bokeh.plotting import figure
from bokeh.embed import components
from bokeh.models import DatetimeTickFormatter, Legend
from bokeh.palettes import Plasma256, linear_palette
from flask_login import current_user
from .database import db
from .database import Transaction, Category, Account
from sqlalchemy.sql import func
from collections import OrderedDict
from numpy import pi

def graph(report_name):
    """Report graph."""
    if report_name == "Expenses by Category":
        graph = ExpensesByCategoryPieGraph()
    elif report_name == "Income by Category":
        graph = IncomeByCategoryPieGraph()
    elif report_name == "Cash Flow":
        graph = CashFlowLineGraph()
    elif report_name == "Account Balances":
        graph = AccountBalancesLineGraph()
    return graph.get_html()

class PieGraph():
    """Pie graph."""

    def __init__(self):
        """Initialise."""
        self.data = []

    def get_html(self):
        """Get HTML components."""
        if self.data:
            amounts = [0]
            labels = []
            for row in self.data:
                labels.append(row[0])
                amounts.append(row[1])
        else:
            labels = ["No Data"]
            amounts = [0, 100]
        total = sum(amounts)
        amounts = [value / total for value in amounts]
        running_totals = []
        running_totals.append(amounts[0])
        running_totals.append(amounts[1])
        for num, value in enumerate(amounts):
            if num == 0 or num == 1:
                continue
            running_totals.append(sum(amounts[0:num + 1]))
        start_angles = [2 * pi * value for value in running_totals[:-1]]
        end_angles = [2 * pi * value for value in running_totals[1:]]
        num_colors = len(labels)
        if num_colors <= 256:
            colors = linear_palette(Plasma256, num_colors)
        else:
            colors = linear_palette(Plasma256, 256) * int(num_colors / 256 + 1)
        pie_chart = figure(
            x_range=(-1, 1), y_range=(-1, 1), plot_width=800, plot_height=600,
            logo=None)
        pie_chart.xaxis.visible = False
        pie_chart.yaxis.visible = False

        legend_items = []
        for num, label in enumerate(labels):
            wedge = pie_chart.wedge(
                x=0, y=0, radius=1, start_angle=start_angles[num],
                end_angle=end_angles[num], color=colors[num])
            percent = ' ' + str(round(amounts[num+1] * 100, 1)) + '%'
            legend_items.append((label + percent, [wedge]))

        legend = Legend(items=legend_items[::-1], location=(0, 0))
        pie_chart.add_layout(legend, 'right')

        script, div = components(pie_chart)
        return script, div

class ExpensesByCategoryPieGraph(PieGraph):
    """Expenses by category pie graph."""

    def __init__(self):
        """Perform database query."""
        super().__init__()
        self.data = (
            db.session.query(Category.catname, func.sum(Transaction.amount))
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .filter(Category.cattype == 'Expense')
            .group_by(Category.catname)
            .order_by(func.sum(Transaction.amount))
            .all())

class IncomeByCategoryPieGraph(PieGraph):
    """Income by category pie graph."""

    def __init__(self):
        """Perform database query."""
        super().__init__()
        self.data = (
            db.session.query(Category.catname, func.sum(Transaction.amount))
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .filter(Category.cattype == 'Income')
            .group_by(Category.catname)
            .order_by(func.sum(Transaction.amount))
            .all())

class LineGraph():
    """
    Line graph.

    Expects data to be { item1: [[x-values], [y-values]],
                         item2: [[x-values], [y-values]],
                         ...                              }
    """

    def __init__(self):
        """Initialise."""
        self.data = OrderedDict()

    def get_html(self):
        """Get HTML components."""
        plot = figure(
            x_axis_type='datetime', x_axis_label='Date', y_axis_label='Amount',
            logo=None)

        DATE_TIME_FORMAT = {
            'days': ['%d/%m/%y'], 'months': ['%m/%Y']}
        plot.xaxis.formatter = DatetimeTickFormatter(**DATE_TIME_FORMAT)
        num_colors = len(self.data)
        if num_colors > 256:
            num_colors = 256
        colors = linear_palette(Plasma256, len(self.data))
        items = []
        if self.data:
            for num, label in enumerate(self.data.keys()):
                dates = self.data[label][0]
                amounts = self.data[label][1]
                num = num % num_colors  # Start colors again if all used
                line = plot.line(
                    dates, amounts, line_color=colors[num], line_width=2)
                items.append((label, [line]))
        plot.sizing_mode = 'scale_width'

        legend = Legend(items=items, location=(0, 0))

        plot.add_layout(legend, 'below')

        script, div = components(plot)
        return script, div

class AccountBalancesLineGraph(LineGraph):
    """Account balances line graph."""

    def __init__(self):
        """Perform database query and populate data structure."""
        super().__init__()
        accounts = (
            db.session.query(Account.accname, Account.accno)
            .filter(Account.id == current_user.id)
            .order_by(Account.accname)
            .all())
        for accname, accno in accounts:
            balance = (
                db.session.query(func.sum(Account.balance))
                .filter(Account.id == current_user.id)
                .filter(Account.accname == accname)
                .first()[0] / 100.0)
            transactions = (
                db.session.query(
                    Transaction.date, Transaction.amount, Category.cattype)
                .filter(Transaction.id == current_user.id)
                .filter(Transaction.accno == accno)
                .filter(Transaction.catno == Category.catno)
                .order_by(Transaction.date)
                .all())
            dates = []
            amounts = []
            for date, amount, cattype in transactions:
                if cattype == 'Expense' or cattype == 'Transfer Out':
                    balance -= amount / 100.0
                else:
                    balance += amount / 100.0
                dates.append(date)
                amounts.append(balance)
            self.data[accname] = [dates, amounts]

class CashFlowLineGraph(LineGraph):
    """Cash flow line graph."""

    def __init__(self):
        """Perform database query and populate data structure."""
        super().__init__()
        balance = (
            db.session.query(func.sum(Account.balance))
            .filter(Account.id == current_user.id)
            .first()[0] / 100.0)
        transactions = (
            db.session.query(
                Transaction.date, Transaction.amount, Category.cattype)
            .filter(Transaction.id == current_user.id)
            .filter(Transaction.catno == Category.catno)
            .order_by(Transaction.date)
            .all())
        dates = []
        amounts = []
        for date, amount, cattype in transactions:
            if cattype == 'Expense' or cattype == 'Transfer Out':
                balance -= amount / 100.0
            else:
                balance += amount / 100.0
            dates.append(date)
            amounts.append(balance)
        self.data['Net Total'] = [dates, amounts]
<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Avery Wallis
Credit: Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))
l=(int(len(string)))
print('You entered "'+ string + '". Now jumble it:')
the = list(string)
spil = string.split(' ')
print()

for x in range(1,l+1):
    print(string[-x],end="")
print()

f=list(reverse(spil))
for word in f:
    print(word,end="")
print()

for x in range(l,1,-1):
    if string[-x]==" ":
        for y in range(l,x,-1):
            a=str(y)

        print(str(y))
"""
for x in range(l,1,-1):
    if string[-x]==" ":
        for y in range(l,x-1,1):
            print(string[-y+(l-x)],end="")
"""
<commit_msg>Updated from Brython Server: 2/12/2016 2:39:48 PM<commit_after>"""
stringjumble.py
Author: Avery Wallis
Credit: Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))
l=(int(len(string)))
print('You entered "'+ string + '". Now jumble it:')
the = list(string)
spil = string.split(' ')
print()

for x in range(1,l+1):
    print(string[-x],end="")
print()

f=list(reversed(spil))
for word in f:
    print(word,end="")
print()

for x in range(l,1,-1):
    if string[-x]==" ":
        for y in range(l,x,-1):
            a=str(y)

        print(str(y))
"""
for x in range(l,1,-1):
    if string[-x]==" ":
        for y in range(l,x-1,1):
            print(string[-y+(l-x)],end="")
"""
<|endoftext|>


<commit_before>import logging
import re
from fabric.api import env
from fabric.operations import run, get

from job.basejob import BaseJob

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

class Job(BaseJob):
    def __str__(self):
        return "{}[{}]".format(BaseJob.__str__(self), self._get_user_at_host_configuration())

    def execute(self):
        env.host_string = self._get_user_at_host_configuration()
        backup_file_name = "backup.tar.gz"
        log.info("creating backup file remotely for: {}".format(self._get_user_at_host_configuration()))
        run(self._get_create_backup_file_command(backup_file_name), shell=False)
        log.info("downloading backup file: {}".format(self._get_user_at_host_configuration()))
        get(backup_file_name, self._get_output_file_name("WebHost"))

    def _get_user_at_host_configuration(self):
        return self.configuration["user_at_host"]

    def _get_create_backup_file_command(self, backup_file_name):
        list_source_files = "ls -A | grep -Ev \"^($BACKUP_FILE|logs|Maildir|.+\.log)$\""
        ignore_failed_read = "--ignore-failed-read" if self.configuration.get("ignore_failed_read", False) else ""
        excludes = " ".join(
            "--exclude={}".format(exclude) for exclude in self.configuration.get("exclude_patterns  ", "").split(","))
        create_backup_file = "BACKUP_FILE_NAME={} ; " \
                             "rm -f $BACKUP_FILE_NAME ; " \
                             "tar {} {} -zcf $BACKUP_FILE_NAME $({})".format(backup_file_name, ignore_failed_read,
                                                                          excludes, list_source_files)
        return create_backup_file

    def _get_output_file_name(self, job_type):
        user_at_host = self._get_user_at_host_configuration()
        name = re.sub("^.*@", "", user_at_host)
        output_filename = "{}/{}-{}.tar.gz".format(self.output_folder, job_type, name)
        return output_filename

    def _validate_configuration(self, configuration):
        pass
<commit_msg>Fixing some wonky imports<commit_after>import logging
import re
from fabric.api import env
from fabric.operations import run, get

from job.basejob import BaseJob

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

class Job(BaseJob):
    def __str__(self):
        return "{}[{}]".format(BaseJob.__str__(self), self._get_user_at_host_configuration())

    def execute(self):
        env.host_string = self._get_user_at_host_configuration()
        backup_file_name = "backup.tar.gz"
        log.info("creating backup file remotely for: {}".format(self._get_user_at_host_configuration()))
        run(self._get_create_backup_file_command(backup_file_name), shell=False)
        log.info("downloading backup file: {}".format(self._get_user_at_host_configuration()))
        get(backup_file_name, self._get_output_file_name("WebHost"))

    def _get_user_at_host_configuration(self):
        return self.configuration["user_at_host"]

    def _get_create_backup_file_command(self, backup_file_name):
        list_source_files = "ls -A | grep -Ev \"^($BACKUP_FILE|logs|Maildir|.+\.log)$\""
        ignore_failed_read = "--ignore-failed-read" if self.configuration.get("ignore_failed_read", False) else ""
        create_backup_file = "BACKUP_FILE_NAME={} ; " \
                             "rm -f $BACKUP_FILE_NAME ; " \
                             "tar {} -zcf $BACKUP_FILE_NAME $({})".format(backup_file_name, ignore_failed_read,
                                                                          list_source_files)
        return create_backup_file

    def _get_output_file_name(self, job_type):
        user_at_host = self._get_user_at_host_configuration()
        name = re.sub("^.*@", "", user_at_host)
        output_filename = "{}/{}-{}.tar.gz".format(self.output_folder, job_type, name)
        return output_filename

    def _validate_configuration(self, configuration):
        pass
<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Billy
Credit: <sources>

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""

userString = input("Enter a sentene, word or paragraph.")

"""
for i in range(inputLength):
    for userString[:]:
        print (inputLength)
"""

print(len([userString]))<commit_msg>Updated from Brython Server: 3/9/2016 1:43:12 PM<commit_after>"""
stringjumble.py
Author: Billy
Credit: http://stackoverflow.com/questions/18455222/how-to-count-the-number-of-letters-in-a-string-without-the-spaces, David, Ryan

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""

userString = input("Enter a sentene, word or paragraph.")

inputLength = userString.length()

print(inputLength)

"""
for i in range(inputLength):
    for userString[:]:
        print (inputLength)
"""

print(len([userString]))

from collections import Counter
import string

def count_letters(word):
    global count
    wordsList = string.split(word)
    count = Counter()
    for words in wordsList:
        for letters in set(words):
            return count[letters]

word = "The grey old fox is an idiot"
print count_letters(word)<|endoftext|>


<commit_before>"""
stringjumble.py
Author: <your name>
Credit: <sources>

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
x=str(input("Please enter a string of text (the bigger the better): "))
x=list(x)
print(x)
z=0
while z<=len(x):
    print(x[z],end="")
    z+=1<commit_msg>Updated from Brython Server: 2/8/2016 2:25:56 PM<commit_after>"""
stringjumble.py
Author: <your name>
Credit: <sources>

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
x=str(input("Please enter a string of text (the bigger the better): "))
x=list(x)
print(x)
z=0
while z<=len(x):
    y=x[z]
    print(y,end="")
    z+=1<|endoftext|>


<commit_before>from celery import registry
from datetime import datetime
from UserDict import UserDict
from celery.serialization import pickle
import atexit
import errno
import time

schedule = PersistentDict(save_at_exit=True)

class ScheduleEntry(object):
    """An entry in the scheduler.

    :param task: The task class.
    :keyword last_run_at: The time and date when this task was last run.
    :keyword total_run_count: Total number of times this periodic task has
        been executed.

    """

    def __init__(self, task, last_run_at=None, total_run_count=None)
        self.task = task
        self.last_run_at = None
        self.total_run_count = None

    def execute(self):
        try:
            result = self.task.apply_async()
        except Exception, exc:
            print("Couldn't apply scheduled task %s: %s" % (
                self.task.name, exc))
            result = None
        self.last_run_at = datetime.now()
        self.total_run_count += 1
        return result

    def is_due(self):
        run_at = self.last_run_at + self.task.run_every
        return True if datetime.now() > self.last_run_at else return False

class Scheduler(object):
    """Scheduler for periodic tasks.

    :keyword registry: The task registry to use.
    :keyword schedule: The schedule dictionary. Default is the global
        persistent schedule ``celery.beat.schedule``.

    """

    registry = registry.tasks
    data = schedule

    def __init__(self, registry=None, schedule=None):
        self.registry = registry or self.registry
        self.data = schedule or self.data
        self.schedule_registry()

    def run(self):
        """Run the scheduler.

        This runs :meth:`tick` every second in a never-exit loop."""
        while True:
            self.tick()
            time.sleep(1)

    def tick(self):
        """Run a tick, that is one iteration of the scheduler.
        Executes all due tasks."""
        return [(entry.task, entry.execute())
                    for entry in self.get_due_tasks()]

    def get_due_tasks(self):
        """Get all the schedule entries that are due to execution."""
        return filter(lambda entry: entry.is_due(), self.schedule.values())

    def schedule_registry(self):
        """Add the current contents of the registry to the schedule."""
        periodic_tasks = self.registry.get_all_periodic()
        schedule = dict((name, tasktimetuple(task))
                            for name, task in periodic_tasks.items())
        self.schedule = dict(schedule, **self.schedule)

    @property
    def schedule(self):
        return self.data

class PersistentDict(UserDict):
    """Dictionary that can be stored to disk.

    :param filename: Name of the file to save to.

    :keyword initial_data: Initial dict to start with.
    :keyword save_at_exit: Register an atexit handler to automatically save
        the data when the program exits (not safe, but as an extra precaution)
    :keyword encoding: The encoding to write the file with, default is
        ``"zlib"``.

    """
    encoding = "zlib"
    save_at_exit = False

    def __init__(self, filename, initial_data=None, save_at_exit=None,
            encoding=None):
        self.data = initial_data or {}
        self.filename = filename
        self.encoding = encoding
        if save_at_exit is not None:
            self.save_at_exit = save_at_exit
        self.reload()
        self._saved = False
        self.save_at_exit and self.register_atexit()

    def reload(self):
        """Reload data from disk."""
        persisted_data = self._read_file(self.filename, self.encoding)
        self.data = dict(self.data, **persisted_data)

    def save(self):
        """Save data to disk."""
        self._saved = True
        encoded = pickle.dump(fh, self.data).encode(self.encoding)

        fh = open(self.filename, "w")
        try:
            fh.write(encoded)
        finally:
            fh.close()

    def register_atexit(self):
        """Register an atexit handler to save data to disk when the
        program terminates."""
        atexit.register(self._save_atexit)

    def _save_atexit(self):
        self._saved or self.save()

    def _read_file(self, filename, encoding):
        try:
            fh = open(filename)
        except IOError, exc:
            if exc.errno == errno.ENOENT:
                return
            raise

        try:
            return pickle.loads(fh.read().decode(encoding))
        finally:
            fh.close()

<commit_msg>Use shelve instead of PersistentDict<commit_after>from UserDict import UserDict
from datetime import datetime
from celery import registry
import shelve
import atexit
import time

schedule = shelve.open(filename="celerybeat-schedule")
atexit.register(schedule.close)

class ScheduleEntry(object):
    """An entry in the scheduler.

    :param task: The task class.
    :keyword last_run_at: The time and date when this task was last run.
    :keyword total_run_count: Total number of times this periodic task has
        been executed.

    """

    def __init__(self, task, last_run_at=None, total_run_count=None)
        self.task = task
        self.last_run_at = None
        self.total_run_count = None

    def execute(self):
        try:
            result = self.task.apply_async()
        except Exception, exc:
            print("Couldn't apply scheduled task %s: %s" % (
                self.task.name, exc))
            result = None
        self.last_run_at = datetime.now()
        self.total_run_count += 1
        return result

    def is_due(self):
        run_at = self.last_run_at + self.task.run_every
        return True if datetime.now() > self.last_run_at else return False

class Scheduler(UserDict):
    """Scheduler for periodic tasks.

    :keyword registry: The task registry to use.
    :keyword schedule: The schedule dictionary. Default is the global
        persistent schedule ``celery.beat.schedule``.

    """

    registry = registry.tasks
    data = schedule

    def __init__(self, registry=None, schedule=None):
        self.registry = registry or self.registry
        self.data = schedule or self.data
        self.schedule_registry()

    def run(self):
        """Run the scheduler.

        This runs :meth:`tick` every second in a never-exit loop."""
        try:
            while True:
                self.tick()
                time.sleep(1)
        finally:
            self.schedule.close()

    def tick(self):
        """Run a tick, that is one iteration of the scheduler.
        Executes all due tasks."""
        return [(entry.task, entry.execute())
                    for entry in self.get_due_tasks()]

    def get_due_tasks(self):
        """Get all the schedule entries that are due to execution."""
        return filter(lambda entry: entry.is_due(), self.schedule.values())

    def schedule_registry(self):
        """Add the current contents of the registry to the schedule."""
        periodic_tasks = self.registry.get_all_periodic()
        schedule = dict((name, tasktimetuple(task))
                            for name, task in periodic_tasks.items())
        self.schedule = dict(schedule, **self.schedule)

    @property
    def schedule(self):
        return self.data
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from datetime import timedelta

from django.utils.encoding import python_2_unicode_compatible
from django.utils import timezone

from django.db import models
from django_countries.fields import CountryField

from django.template.loader import get_template
from django.template import Context

from core.models import User

from jobs.community_mails import send_job_mail, send_meetup_mail

class PublishFlowManager(models.Manager):

    def get_queryset(self):
        now = timezone.now().date().strftime("%Y-%m-%d")
        return super(PublishFlowManager, self).get_queryset().extra(
            select={'not_expired': "coalesce(expiration_date > '%s', 1)" % now})

class VisiblePublishFlowManager(PublishFlowManager):

    def get_queryset(self):
        return super(VisiblePublishFlowManager, self).get_queryset().filter(
            review_status=PublishFlowModel.PUBLISHED,
            expiration_date__gte=timezone.now()
        )

class PublishFlowModel(models.Model):
    """
    An abstract class model that handles all logic related to publishing
    an item that needs to be reviewed
    """
    OPEN = 'OPN'
    UNDER_REVIEW = 'URE'
    READY_TO_PUBLISH = 'RTP'
    REJECTED = 'REJ'
    PUBLISHED = 'PUB'
    STATUSES = (
        (OPEN, 'Open'),
        (UNDER_REVIEW, 'Under review'),
        (READY_TO_PUBLISH, 'Ready to publish'),
        (REJECTED, 'Rejected'),
        (PUBLISHED, 'Published'),
    )

    reviewer = models.ForeignKey(
        User,
        related_name="%(app_label)s_%(class)s_related",
        blank=True,
        null=True,
        on_delete=models.SET_NULL
    )
    review_status = models.CharField(max_length=3, choices=STATUSES, default=OPEN)
    message_to_organisation = models.TextField(
        blank=True,
        null=True,
        help_text="Write your message to the company/organisation here."
    )
    internal_comment = models.TextField(
        blank=True,
        null=True,
        help_text="Write you comments here. They won't be sent to "
                  "the company/organisation."
    )
    published_date = models.DateTimeField(blank=True, null=True)
    created = models.DateTimeField(auto_now_add=True)
    expiration_date = models.DateField(
        blank=True,
        null=True,
        help_text="Automatically is set 60 days from posting. You can"
                  " override this."
    )

    objects = PublishFlowManager()
    visible_objects = VisiblePublishFlowManager()

    class Meta:
        abstract = True
        ordering = ['-published_date']

    def is_ready_to_publish(self):
        return self.review_status == self.READY_TO_PUBLISH

    def assign(self, user):
        assert self.review_status == self.OPEN
        self.reviewer = user
        self.review_status = self.UNDER_REVIEW
        self.save()

    def unassign(self):
        assert self.review_status == self.UNDER_REVIEW
        self.reviewer = None
        self.review_status = self.OPEN
        self.save()

    def accept(self):
        assert self.review_status == self.UNDER_REVIEW
        self.review_status = self.READY_TO_PUBLISH
        self.save()

    def reject(self):
        assert self.review_status in [self.UNDER_REVIEW, self.READY_TO_PUBLISH, self.PUBLISHED]
        self.review_status = self.REJECTED
        self.published_date = None
        self.save()
        subject = '{0} was rejected.'.format(self.title)
        model_name = self._meta.model_name
        context = Context({
                    'status': self.get_review_status_display(),
                    'option': model_name,
                    'message_to_organisation': self.message_to_organisation,
                })
        message_plain = get_template(
            'jobs/email_templates/status.txt').render(context)
        message_html = get_template(
            'jobs/email_templates/status.html').render(context)
        recipient = self.contact_email
        if model_name == 'job':
            send_job_mail(
                    subject,
                    message_plain,
                    message_html,
                    recipient
                )
        elif model_name == 'meetup':
            send_meetup_mail(
                subject,
                message_plain,
                message_html,
                recipient,
            )

    def restore(self, user):
        assert self.review_status == self.REJECTED
        self.reviewer = user
        self.review_status = self.UNDER_REVIEW
        self.save()

    def publish(self):
        assert self.is_ready_to_publish()
        self.published_date = timezone.now()
        # the line below covers the situation when there is no expiration date
        # set or when somebody wants to republish an expired post but
        # hasn't changed the expiration date
        if not self.expiration_date or self.expiration_date < self.published_date.date():
            self.expiration_date = self.published_date + timedelta(60)
        self.review_status = self.PUBLISHED
        self.save()
        subject = '{0} is now published.'.format(self.title)
        model_name = self._meta.model_name
        context = Context({
                    'status': self.get_review_status_display(),
                    'option': model_name,
                    'message_to_organisation': self.message_to_organisation,
                })
        message_plain = get_template(
            'jobs/email_templates/status.txt').render(context)
        message_html = get_template(
            'jobs/email_templates/status.html').render(context)
        recipient = self.contact_email
        if model_name == 'job':
            send_job_mail(
                    subject,
                    message_plain,
                    message_html,
                    recipient
                )
        elif model_name == 'meetup':
            send_meetup_mail(
                subject,
                message_plain,
                message_html,
                recipient,
            )

@python_2_unicode_compatible
class Job(PublishFlowModel):
    title = models.CharField(max_length=255)
    company = models.CharField(max_length=255)
    website = models.URLField(
        help_text="Link to your offer or company website.",
        blank=True,
        null=True
    )
    contact_email = models.EmailField(max_length=255)
    cities = models.CharField(max_length=255)
    country = CountryField()
    description = models.TextField()
    remote_work = models.BooleanField(
        default=False,
    )
    relocation = models.BooleanField(
        default=False,
    )

    class Meta(PublishFlowModel.Meta):
        unique_together = (("company", "title"),)

    def __str__(self):
        return "{0}, {1}".format(self.title, self.company)

@python_2_unicode_compatible
class Meetup(PublishFlowModel):

    MEETUP = 'MEET'
    CONFERENCE = 'CONF'
    WORKSHOP = 'WORK'
    MEETUP_TYPES = (
        (MEETUP, 'meetup'),
        (CONFERENCE, 'conference'),
        (WORKSHOP, 'workshop'),
    )

    title = models.CharField(max_length=255)
    organisation = models.CharField(
        max_length=255,
        blank=True,
        null=True,
    )
    meetup_type = models.CharField(
        max_length=4,
        choices=MEETUP_TYPES,
        default=MEETUP
    )
    contact_email = models.EmailField(max_length=255)
    website = models.URLField(
        help_text="Link to your meetup or organisation website.",
        blank=True,
        null=True
    )
    city = models.CharField(max_length=255)
    country = CountryField()
    description = models.TextField()
    is_recurring = models.BooleanField(
        default=False,
    )
    #TODO this field should be required if the is_recurring is True
    recurrence = models.CharField(
        max_length=255,
        blank=True,
        null=True,
        help_text="Provide details of recurrence if applicable."
    )
    meetup_start_date = models.DateTimeField(
        null=True,
        help_text="If this is a recurring meetup/event, please enter a start date.\
            Date format: YYYY-MM-DD"
    )
    meetup_end_date = models.DateTimeField(
        null=True,
        blank=True,
        help_text="Date format: YYYY-MM-DD"
    )

    class Meta(PublishFlowModel.Meta):
        unique_together = (("title", "city"),)

    def __str__(self):
        return "{0}, {1}".format(self.title, self.city)
<commit_msg>Corrected ProgrammingError COALESCE types<commit_after>from __future__ import unicode_literals
from datetime import timedelta

from django.utils.encoding import python_2_unicode_compatible
from django.utils import timezone

from django.db import models
from django_countries.fields import CountryField

from django.template.loader import get_template
from django.template import Context

from core.models import User

from jobs.community_mails import send_job_mail, send_meetup_mail

class PublishFlowManager(models.Manager):

    def get_queryset(self):
        now = timezone.now().date().strftime("%Y-%m-%d")
        # 1=1 to ensure compatibility between different DBs
        # this is to fix the error:
        # 'COALESCE types boolean and integer cannot be matched'
        return super(PublishFlowManager, self).get_queryset().extra(
            select={'not_expired': "coalesce(expiration_date > '%s', 1=1)" % now})

class VisiblePublishFlowManager(PublishFlowManager):

    def get_queryset(self):
        return super(VisiblePublishFlowManager, self).get_queryset().filter(
            review_status=PublishFlowModel.PUBLISHED,
            expiration_date__gte=timezone.now()
        )

class PublishFlowModel(models.Model):
    """
    An abstract class model that handles all logic related to publishing
    an item that needs to be reviewed
    """
    OPEN = 'OPN'
    UNDER_REVIEW = 'URE'
    READY_TO_PUBLISH = 'RTP'
    REJECTED = 'REJ'
    PUBLISHED = 'PUB'
    STATUSES = (
        (OPEN, 'Open'),
        (UNDER_REVIEW, 'Under review'),
        (READY_TO_PUBLISH, 'Ready to publish'),
        (REJECTED, 'Rejected'),
        (PUBLISHED, 'Published'),
    )

    reviewer = models.ForeignKey(
        User,
        related_name="%(app_label)s_%(class)s_related",
        blank=True,
        null=True,
        on_delete=models.SET_NULL
    )
    review_status = models.CharField(max_length=3, choices=STATUSES, default=OPEN)
    message_to_organisation = models.TextField(
        blank=True,
        null=True,
        help_text="Write your message to the company/organisation here."
    )
    internal_comment = models.TextField(
        blank=True,
        null=True,
        help_text="Write you comments here. They won't be sent to "
                  "the company/organisation."
    )
    published_date = models.DateTimeField(blank=True, null=True)
    created = models.DateTimeField(auto_now_add=True)
    expiration_date = models.DateField(
        blank=True,
        null=True,
        help_text="Automatically is set 60 days from posting. You can"
                  " override this."
    )

    objects = PublishFlowManager()
    visible_objects = VisiblePublishFlowManager()

    class Meta:
        abstract = True
        ordering = ['-published_date']

    def is_ready_to_publish(self):
        return self.review_status == self.READY_TO_PUBLISH

    def assign(self, user):
        assert self.review_status == self.OPEN
        self.reviewer = user
        self.review_status = self.UNDER_REVIEW
        self.save()

    def unassign(self):
        assert self.review_status == self.UNDER_REVIEW
        self.reviewer = None
        self.review_status = self.OPEN
        self.save()

    def accept(self):
        assert self.review_status == self.UNDER_REVIEW
        self.review_status = self.READY_TO_PUBLISH
        self.save()

    def reject(self):
        assert self.review_status in [self.UNDER_REVIEW, self.READY_TO_PUBLISH, self.PUBLISHED]
        self.review_status = self.REJECTED
        self.published_date = None
        self.save()
        subject = '{0} was rejected.'.format(self.title)
        model_name = self._meta.model_name
        context = Context({
                    'status': self.get_review_status_display(),
                    'option': model_name,
                    'message_to_organisation': self.message_to_organisation,
                })
        message_plain = get_template(
            'jobs/email_templates/status.txt').render(context)
        message_html = get_template(
            'jobs/email_templates/status.html').render(context)
        recipient = self.contact_email
        if model_name == 'job':
            send_job_mail(
                    subject,
                    message_plain,
                    message_html,
                    recipient
                )
        elif model_name == 'meetup':
            send_meetup_mail(
                subject,
                message_plain,
                message_html,
                recipient,
            )

    def restore(self, user):
        assert self.review_status == self.REJECTED
        self.reviewer = user
        self.review_status = self.UNDER_REVIEW
        self.save()

    def publish(self):
        assert self.is_ready_to_publish()
        self.published_date = timezone.now()
        # the line below covers the situation when there is no expiration date
        # set or when somebody wants to republish an expired post but
        # hasn't changed the expiration date
        if not self.expiration_date or self.expiration_date < self.published_date.date():
            self.expiration_date = self.published_date + timedelta(60)
        self.review_status = self.PUBLISHED
        self.save()
        subject = '{0} is now published.'.format(self.title)
        model_name = self._meta.model_name
        context = Context({
                    'status': self.get_review_status_display(),
                    'option': model_name,
                    'message_to_organisation': self.message_to_organisation,
                })
        message_plain = get_template(
            'jobs/email_templates/status.txt').render(context)
        message_html = get_template(
            'jobs/email_templates/status.html').render(context)
        recipient = self.contact_email
        if model_name == 'job':
            send_job_mail(
                    subject,
                    message_plain,
                    message_html,
                    recipient
                )
        elif model_name == 'meetup':
            send_meetup_mail(
                subject,
                message_plain,
                message_html,
                recipient,
            )

@python_2_unicode_compatible
class Job(PublishFlowModel):
    title = models.CharField(max_length=255)
    company = models.CharField(max_length=255)
    website = models.URLField(
        help_text="Link to your offer or company website.",
        blank=True,
        null=True
    )
    contact_email = models.EmailField(max_length=255)
    cities = models.CharField(max_length=255)
    country = CountryField()
    description = models.TextField()
    remote_work = models.BooleanField(
        default=False,
    )
    relocation = models.BooleanField(
        default=False,
    )

    class Meta(PublishFlowModel.Meta):
        unique_together = (("company", "title"),)

    def __str__(self):
        return "{0}, {1}".format(self.title, self.company)

@python_2_unicode_compatible
class Meetup(PublishFlowModel):

    MEETUP = 'MEET'
    CONFERENCE = 'CONF'
    WORKSHOP = 'WORK'
    MEETUP_TYPES = (
        (MEETUP, 'meetup'),
        (CONFERENCE, 'conference'),
        (WORKSHOP, 'workshop'),
    )

    title = models.CharField(max_length=255)
    organisation = models.CharField(
        max_length=255,
        blank=True,
        null=True,
    )
    meetup_type = models.CharField(
        max_length=4,
        choices=MEETUP_TYPES,
        default=MEETUP
    )
    contact_email = models.EmailField(max_length=255)
    website = models.URLField(
        help_text="Link to your meetup or organisation website.",
        blank=True,
        null=True
    )
    city = models.CharField(max_length=255)
    country = CountryField()
    description = models.TextField()
    is_recurring = models.BooleanField(
        default=False,
    )
    #TODO this field should be required if the is_recurring is True
    recurrence = models.CharField(
        max_length=255,
        blank=True,
        null=True,
        help_text="Provide details of recurrence if applicable."
    )
    meetup_start_date = models.DateTimeField(
        null=True,
        help_text="If this is a recurring meetup/event, please enter a start date.\
            Date format: YYYY-MM-DD"
    )
    meetup_end_date = models.DateTimeField(
        null=True,
        blank=True,
        help_text="Date format: YYYY-MM-DD"
    )

    class Meta(PublishFlowModel.Meta):
        unique_together = (("title", "city"),)

    def __str__(self):
        return "{0}, {1}".format(self.title, self.city)
<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Payton Stearns
Credit: Avery, Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
ost = str(input("Please enter a string of text (the bigger the better): "))
print("")
print("You entered: '" + ost + "'. Now Jumble it: ")
l = list(ost)
sost = ost.split(' ')

z=0
while z<len(l):
    print(str(l[len(l)-z-1]), end="")
    z=z+1
print("")
print()
f = list(reversed(sost))
for word in f:
    print(word, end=" ")
print("")
print()

for word in sost:
    q=0
    while q<len(word):
        r = (str(word[len(word)-q-1]), end"")
        print(r, end="")
        q=q+1<commit_msg>Updated from Brython Server: 2/10/2016 7:56:45 PM<commit_after>"""
stringjumble.py
Author: Payton Stearns
Credit: Avery, Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
ost = str(input("Please enter a string of text (the bigger the better): "))
print("")
print("You entered: '" + ost + "'. Now Jumble it: ")
l = list(ost)
sost = ost.split(' ')

z=0
while z<len(l):
    print(str(l[len(l)-z-1]), end="")
    z=z+1
print("")
print()
f = list(reversed(sost))
for word in f:
    print(word, end=" ")
print("")
print()

for word in sost:
    q=0
    while q<len(word):
        r = (str(word[len(word)-q-1]), "")
        print(r, end="")
        q=q+1<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""multi process and multi greenlet spawn"""

from gevent.queue import Queue
from functools import partial
import gevent
import gipc
import random
import time
import logging

import sys; sys.modules.pop("threading", None)
from gevent import monkey; monkey.patch_all()
from .dictqueue import DictQueue

logger = logging.getLogger("pglet")
logger = logging

def id_generator():
    i = 0
    wall = 1 << 31
    while True:
        i += 1
        if i > wall:
            i = 1
        yield i

class Delay(object):
    """future对象: PPool的spawn返回的future对象"""

    def __init__(self, ppool, task_id):
        self.ppool = ppool
        self.task_id = task_id

    def get(self, block=True, timeout=None):
        """在这里,block不能为False"""
        return self.ppool.results.get(self.task_id, block, timeout)

class PPool(object):
    """process pool + gevent spawn
    : 1.必须放在入口文件的最开始处调用PPool实例的init函数, 因为子进程会复制主进程空间(包括已有协程)
    : 2.如果需要子进程也包含某些模块/全局变量,可以在模块/全局变量导入后再调用init函数,原则是不能复制过多主进程空间
    Example:
    from share import ppool; ppool.init()
    import sys; sys.modules.pop("threading", None)
    from gevent import monkey; monkey.patch_all()
    import gevent

    def foo(a):
        print a

    ppool.spawn_sub(foo, "abc")

    gevent.wait()
    """

    def __init__(self, process_size=1):
        """
        @param process_size: the process pool size
        """
        self.id_generator = id_generator()
        self.process_size = process_size
        self.parent_pipe_ends = []         # 存放process_size个双向pipe的父进程端
        self.child_pipe_ends = []          # 存放process_size个双向pipe的子进程端
        self.processes = []                # 存放process_size个子进程
        self.results = DictQueue()         # 在阻塞模式下存放结果, {task_id:DictQueue()}
        self.loop_get_result_glets = []    # 存放 loop_get_result 产生的process_size个greenlet

    def init(self):
        for _ in xrange(self.process_size):
            child_pipe_end, parent_pipe_end =  gipc.pipe(duplex=True)
            self.child_pipe_ends.append(child_pipe_end)
            self.parent_pipe_ends.append(parent_pipe_end)
            p = gipc.start_process(target=self.process_target, args=(child_pipe_end, ), daemon=True)
            self.processes.append(p)
            #p.join()

            self.loop_get_result()

    def loop_get_result(self):
        """循环读取子进程返回结果"""
        self.n = 0 # 用于测试性能,发布时删除
        def loop(p):
            while 1:
                try:
                    k, v = p.get()
                    if k == -9: # 用于测试性能,发布时删除, =-9时表示该子进程joinall结束
                        self.n += 1
                        if self.n == self.process_size:
                            print "join all completed!!!", time.time()
                        continue
                    self.results.put(k, v, override=True, timeout=None)
                except Exception as e:
                    logger.warning(str(e))
                    break

        for p in self.parent_pipe_ends:
            g = gevent.spawn(loop, p)
            self.loop_get_result_glets.append(g)

    def process_target(self, child_pipe_end):
        """子进程空间"""
        def callback(g, task_id):
            child_pipe_end.put([task_id, g.value])

        taskqs = [] # 用于测试,发布时删除

        def loop(child_pipe_end, taskqs):
            while 1:
                try:
                    f, args, kwargs, task_id = child_pipe_end.get()
                    if task_id >= 0:
                        g = gevent.spawn(f, *args, **kwargs)
                        cb = partial(callback, task_id=task_id)
                        g.link(cb)
                    elif task_id == -1: # 用于测试性能,发布时删除, =-1时表示joinall开始
                        taskqs.append([f, args])
                    elif task_id == -2: # 用于测试性能,发布时删除, =-2时表示joinall结束
                        gevent.joinall([gevent.spawn(f, *args) for f, args in taskqs])
                        child_pipe_end.put([-9, None])
                    else:
                        gevent.spawn(f, *args, **kwargs)
                except Exception as e:
                    logger.warning(str(e))
                    break

        loop(child_pipe_end, taskqs)

    def _benchmark_join_start(self, f, ts=[]):
        parent_pipe_end = self.select_pipe_writer()
        [parent_pipe_end.put([f, args, None, -1]) for args in ts]

    def _benchmark_join_end(self):
        for p in self.parent_pipe_ends:
            p.put([None, None, None, -2])

    def select_pipe_writer(self,  sn=None):
        if sn is not None:
            return self.parent_pipe_ends[sn]
        return random.choice(self.parent_pipe_ends)

    def _spawn(self, f, args=tuple(), kwargs={}, future=False, timeout=None, sn=None):
        """和gevent.spawn的区别是,如果block=False 就不返回任何值(gevent.spawn返回greenlet),如果block=True,返回f的结果值,而不是greenlet
        @param f: func
        @param args, kwargs: f的参数, ***必须能够pickle序列化***
        @param: future: 是否返回future对象(future.get阻塞获取结果)
        """
        parent_pipe_end = self.select_pipe_writer(sn)
        if future:
            task_id = self.id_generator.next()
            parent_pipe_end.put([f, args, kwargs, task_id])
            delay = Delay(self, task_id)
            return delay
        else:
            parent_pipe_end.put([f, args, kwargs, None])

    def spawn(self, f, *args, **kwargs):
        """f的参数和返回值***必须能够pickle序列化***
        @return: 返回future对象,future.get()来阻塞获取结果
        """
        return self._spawn(f, args, kwargs, True, None)

    def spawn_sub(self, f, *args, **kwargs):
        """f的参数***必须能够pickle序列化***
        @return: 无 (即:是过程sub)
        """
        return self._spawn(f, args, kwargs, False, None)

    def close(self):
        """不再使用时,必须手动调用"""
        try:
            [p.terminate() for p in self.processes]          # 结束子进程
            [g.kill() for g in self.loop_get_result_glets]   # 结束取结果的协程
            [p.close() for p in self.parent_pipe_ends]       # 管道一端关闭即可
        except Exception as e:
            logger.warning(str(e))

    def __del__(self):
        self.close()

if __name__ == "__main__":
    def x(a):
        return a

    try:
        ppool = PPool(2)
        ppool.init()
        print ppool.spawn(x, "abc").get()
        print ppool.spawn_sub(x, "def")
        ppool.close()
        gevent.wait()
    except KeyboardInterrupt:
        ppool.close()

        <commit_msg>benchmark<commit_after># -*- coding: utf-8 -*-

"""multi process and multi greenlet spawn"""

from gevent.queue import Queue
from functools import partial
import gevent
import gipc
import random
import time
import logging

import sys; sys.modules.pop("threading", None)
from gevent import monkey; monkey.patch_all()
from .dictqueue import DictQueue

logger = logging.getLogger("pglet")
logger = logging

def id_generator():
    i = 0
    wall = 1 << 31
    while True:
        i += 1
        if i > wall:
            i = 1
        yield i

class Delay(object):
    """future对象: PPool的spawn返回的future对象"""

    def __init__(self, ppool, task_id):
        self.ppool = ppool
        self.task_id = task_id

    def get(self, block=True, timeout=None):
        """在这里,block不能为False"""
        return self.ppool.results.get(self.task_id, block, timeout)

class PPool(object):
    """process pool + gevent spawn
    : 1.必须放在入口文件的最开始处调用PPool实例的init函数, 因为子进程会复制主进程空间(包括已有协程)
    : 2.如果需要子进程也包含某些模块/全局变量,可以在模块/全局变量导入后再调用init函数,原则是不能复制过多主进程空间
    Example:
    from share import ppool; ppool.init()
    import sys; sys.modules.pop("threading", None)
    from gevent import monkey; monkey.patch_all()
    import gevent

    def foo(a):
        print a

    ppool.spawn_sub(foo, "abc")

    gevent.wait()
    """

    def __init__(self, process_size=1):
        """
        @param process_size: the process pool size
        """
        self.id_generator = id_generator()
        self.process_size = process_size
        self.parent_pipe_ends = []         # 存放process_size个双向pipe的父进程端
        self.child_pipe_ends = []          # 存放process_size个双向pipe的子进程端
        self.processes = []                # 存放process_size个子进程
        self.results = DictQueue()         # 在阻塞模式下存放结果, {task_id:DictQueue()}
        self.loop_get_result_glets = []    # 存放 loop_get_result 产生的process_size个greenlet

    def init(self):
        for _ in xrange(self.process_size):
            child_pipe_end, parent_pipe_end =  gipc.pipe(duplex=True)
            self.child_pipe_ends.append(child_pipe_end)
            self.parent_pipe_ends.append(parent_pipe_end)
            p = gipc.start_process(target=self.process_target, args=(child_pipe_end, ), daemon=True)
            self.processes.append(p)
            #p.join()

            self.loop_get_result()

    def loop_get_result(self):
        """循环读取子进程返回结果"""
        self.n = 0 # 用于测试性能,发布时删除
        def loop(p):
            while 1:
                try:
                    k, v = p.get()
                    if k == -9: # 用于测试性能,发布时删除, =-9时表示该子进程joinall结束
                        self.n += 1
                        if self.n == self.process_size:
                            print "join all completed!!!", time.time()
                        continue
                    self.results.put(k, v, override=True, timeout=None)
                except Exception as e:
                    logger.warning(str(e))
                    break

        for p in self.parent_pipe_ends:
            g = gevent.spawn(loop, p)
            self.loop_get_result_glets.append(g)

    def process_target(self, child_pipe_end):
        """子进程空间"""
        def callback(g, task_id):
            child_pipe_end.put([task_id, g.value])

        taskqs = [] # 用于测试,发布时删除

        def loop(child_pipe_end, taskqs):
            while 1:
                try:
                    f, args, kwargs, task_id = child_pipe_end.get()
                    if task_id >= 0:
                        g = gevent.spawn(f, *args, **kwargs)
                        cb = partial(callback, task_id=task_id)
                        g.link(cb)
                    elif task_id == -1: # 用于测试性能,发布时删除, =-1时表示joinall开始
                        taskqs.append([f, args])
                    elif task_id == -2: # 用于测试性能,发布时删除, =-2时表示joinall结束
                        gevent.joinall([gevent.spawn(f, args) for f, args in taskqs])
                        child_pipe_end.put([-9, None])
                    else:
                        gevent.spawn(f, *args, **kwargs)
                except Exception as e:
                    logger.warning(str(e))
                    break

        loop(child_pipe_end, taskqs)

    def _benchmark_join_start(self, f, ts=[]):
        parent_pipe_end = self.select_pipe_writer()
        [parent_pipe_end.put([f, args, None, -1]) for args in ts]

    def _benchmark_join_end(self):
        for p in self.parent_pipe_ends:
            p.put([None, None, None, -2])

    def select_pipe_writer(self,  sn=None):
        if sn is not None:
            return self.parent_pipe_ends[sn]
        return random.choice(self.parent_pipe_ends)

    def _spawn(self, f, args=tuple(), kwargs={}, future=False, timeout=None, sn=None):
        """和gevent.spawn的区别是,如果block=False 就不返回任何值(gevent.spawn返回greenlet),如果block=True,返回f的结果值,而不是greenlet
        @param f: func
        @param args, kwargs: f的参数, ***必须能够pickle序列化***
        @param: future: 是否返回future对象(future.get阻塞获取结果)
        """
        parent_pipe_end = self.select_pipe_writer(sn)
        if future:
            task_id = self.id_generator.next()
            parent_pipe_end.put([f, args, kwargs, task_id])
            delay = Delay(self, task_id)
            return delay
        else:
            parent_pipe_end.put([f, args, kwargs, None])

    def spawn(self, f, *args, **kwargs):
        """f的参数和返回值***必须能够pickle序列化***
        @return: 返回future对象,future.get()来阻塞获取结果
        """
        return self._spawn(f, args, kwargs, True, None)

    def spawn_sub(self, f, *args, **kwargs):
        """f的参数***必须能够pickle序列化***
        @return: 无 (即:是过程sub)
        """
        return self._spawn(f, args, kwargs, False, None)

    def close(self):
        """不再使用时,必须手动调用"""
        try:
            [p.terminate() for p in self.processes]          # 结束子进程
            [g.kill() for g in self.loop_get_result_glets]   # 结束取结果的协程
            [p.close() for p in self.parent_pipe_ends]       # 管道一端关闭即可
        except Exception as e:
            logger.warning(str(e))

    def __del__(self):
        self.close()

if __name__ == "__main__":
    def x(a):
        return a

    try:
        ppool = PPool(2)
        ppool.init()
        print ppool.spawn(x, "abc").get()
        print ppool.spawn_sub(x, "def")
        ppool.close()
        gevent.wait()
    except KeyboardInterrupt:
        ppool.close()

        <|endoftext|>


<commit_before><commit_msg>Delete __init__.py<commit_after><|endoftext|>


<commit_before>"""
    lstm (needs modularisation and further refactoring)

    sequence classification of variable lengths through
    Long-short Term Memory (LSTM) which models surrounding context of words.

    Words are represented in 300-dimension vectors to allow generalization over
    semantic redundancy.

    Distributed Representations of Words and Phrases and their Compositionality
    Mikolov et al. 2013
"""
import os

import numpy as np
import tensorflow as tf

from tqdm import tqdm
from datetime import datetime
from sklearn import model_selection, preprocessing

from chatbot.engine import corpus, labels
from chatbot.nlp.embedding import WordEmbedding
from chatbot.nlp.sparse import NLPProcessor
from chatbot.serializers import feed_conversation
from chatbot.settings import (CacheSettings, NLP, FORCE, STATE_SIZE, STEP_SIZE,
                              BATCH_SIZE, MAX_WORDS, MAX_STEPS, VERBOSE)

def resample(docs, labels, sample_size):
    unique_labels = np.unique(labels)
    indice_array = list()
    for label in unique_labels:
        indice_label = np.where(labels == label)[0].tolist()
        indice_array.extend(
            np.random.choice(indice_label, size=sample_size).tolist())
    return (docs[indice_array], labels[indice_array])

def flatten_split_resample(encoded_corpuses, encoded_labels,
                           valid_ratio=.2,
                           sample_size=1000):
    """break documents into sentences and augment, and one-hot encode labels"""

    flattened_docs = list()
    flattened_labels = list()
    for d, l in zip(encoded_corpuses, encoded_labels):
        for sent in d:
            flattened_docs.append(sent)
            flattened_labels.append(l)

    ravelled_corpus = np.array(flattened_docs)
    ravelled_labels = np.array(flattened_labels)

    X_train, X_test, y_train, y_test = \
        model_selection.train_test_split(ravelled_corpus, ravelled_labels,
                                         test_size=valid_ratio,
                                         stratify=ravelled_labels)

    return resample(X_train, y_train, sample_size),\
        resample(X_test, y_test, sample_size)

def size(sequence):
    """produces length vector in shape [batch_size, length] for each padded
    sequence of its true length excluding zero vectors.
    sequence of vectors comes in shape of [batch_size, STEP_SIZE, dimensions]
    """
    flag = tf.sign(tf.reduce_max(tf.abs(sequence), axis=2))
    length = tf.reduce_sum(flag, axis=1)
    return tf.cast(length, tf.int32)

def find_last(outputs, length):
    """zero padded sequences will produce zero vectors after sequence_length
    required using dynamic unrolling dynamic_rnn API. this function locates
    the last non-zero output for calculating logits
    """
    # outputs in shape [BATCH_SIZE, STEP_SIZE, STATE_SIZE], length [BATCH_SIZE]
    b, t, st = tf.unstack(tf.shape(outputs))
    flat_index = tf.range(b) * t + (length - 1)
    zeros = tf.zeros(shape=[b], dtype=tf.int32)
    non_neg_flat_index = tf.maximum(zeros, flat_index)
    flat_outputs = tf.reshape(outputs, [-1, st])
    last_outputs = tf.gather(flat_outputs, non_neg_flat_index)
    return last_outputs

def enqueue(sent_encoded, label_encoded, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued sentence and label pair"""
    sent = tf.convert_to_tensor(sent_encoded, dtype=tf.int32)
    label = tf.convert_to_tensor(label_encoded, dtype=tf.uint8)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[sent, label],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def batch_generator(sent_queue, label_queue, batch_size=None, threads=8):
    return tf.train.shuffle_batch(
                    tensors=[sent_queue, label_queue],
                    batch_size=batch_size,
                    num_threads=threads,
                    capacity=1e3,
                    min_after_dequeue=batch_size,
                    allow_smaller_final_batch=True)

def restore_session(sess, path):
    """restore hard trained model for predicting."""
    eval_saver = \
        tf.train.import_meta_graph(tf.train.latest_checkpoint(path) + '.meta')
    eval_saver.restore(sess, tf.train.latest_checkpoint(path))
    print('{} restored successfully.'.format(tf.train.latest_checkpoint(path)))

def save_session(sess, path, sav):
    """save hard trained model for future predicting."""
    now = datetime.now().strftime('%Y%m%d%H%M%S')
    if not os.path.exists(path):
        os.makedirs(path)
    save_path = sav.save(sess, path + "model_{0}.ckpt".format(now))
    print("Model saved in: {0}".format(save_path))

def train(n, sess, is_train, optimiser, metric, loss, verbose):

    for global_step in tqdm(range(n), unit='step', disable=verbose):
        _, train_accuracy, train_loss = \
            sess.run([optimiser, metric, loss], feed_dict={is_train: True})

        if verbose:
            print("step {0} of {3}, train accuracy: {1:.4f} log loss: {2:.4f}"
                  .format(global_step, train_accuracy, train_loss, n))

        if global_step and global_step % 100 == 0:
            valid_accuracy, valid_loss = sess.run(fetches=[metric, loss])
            print("step {0} of {3}, valid accuracy: {1:.4f} "
                  "log loss: {2:.4f}".format(global_step, valid_accuracy,
                                             valid_loss, n))

    print("step {0} of {0}, train accuray: {1:.4f}, valid accuracy {2:.4f} "
          "log loss: {3:.4f}".format(n, train_accuracy, valid_accuracy,
                                     train_loss))

corpus = NLPProcessor(attrs=NLP).process(corpus)

corpus_encoder = WordEmbedding(top=MAX_WORDS).fit(corpus)
encoded_corpus = corpus_encoder.encode(zero_pad=True, pad_length=STEP_SIZE)

l_encoder = preprocessing.LabelBinarizer().fit(labels)
encoded_labels, classes = l_encoder.transform(labels), l_encoder.classes_

embedding_matrix = corpus_encoder.vectorize()
embed_shape = embedding_matrix.shape

data = flatten_split_resample(encoded_corpus, encoded_labels, valid_ratio=.2)
train_sent, train_label = batch_generator(*enqueue(*data[0]), BATCH_SIZE)
valid_sent, valid_label = batch_generator(*enqueue(*data[1]), BATCH_SIZE)

query = tf.placeholder_with_default(input=valid_sent,
                                    shape=[None, STEP_SIZE],
                                    name='query')
embeddings = tf.placeholder_with_default(input=embedding_matrix,
                                         shape=[None, embed_shape[-1]],
                                         name='embeddings')
is_train = tf.placeholder_with_default(input=False,
                                       shape=[],
                                       name='is_train_or_valid')

feature_feed = tf.cond(is_train, lambda: train_sent, lambda: query)
label_feed = tf.cond(is_train, lambda: train_label, lambda: valid_label)
keep_prob = tf.cond(is_train, lambda: tf.constant(.8), lambda: tf.constant(1.))

W_softmax = tf.get_variable(name='W',
                            shape=[STATE_SIZE, len(labels)],
                            initializer=tf.truncated_normal_initializer(
                                        stddev=0.1))
b_softmax = tf.get_variable(name='b',
                            shape=[len(labels)],
                            initializer=tf.constant_initializer(0.0))

word_vectors = tf.nn.embedding_lookup(embeddings, feature_feed)

with tf.device('/gpu:0'):

    cell = tf.nn.rnn_cell.BasicLSTMCell(STATE_SIZE)
    cell = tf.nn.rnn_cell.DropoutWrapper(cell=cell,
                                         input_keep_prob=keep_prob,
                                         output_keep_prob=keep_prob,
                                         state_keep_prob=keep_prob)
    sent_length = size(word_vectors)
    outputs, final_state = tf.nn.dynamic_rnn(cell=cell,
                                             inputs=word_vectors,
                                             sequence_length=sent_length,
                                             dtype=tf.float32)

    last = find_last(outputs, sent_length)
    logits = tf.matmul(last, W_softmax) + b_softmax

    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                            labels=label_feed)

    loss = tf.reduce_mean(cross_entropy)
    train_step = tf.train.RMSPropOptimizer(1e-3).minimize(loss)

    probs = tf.nn.softmax(logits)
    correct = tf.equal(tf.argmax(probs, 1), tf.argmax(label_feed, 1))
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

    saver = tf.train.Saver(max_to_keep=5, var_list=tf.global_variables())

    init = tf.global_variables_initializer()

    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
    sess.run(init)

with sess.as_default(), tf.device('/cpu:0'):
    try:
        if FORCE:
            raise TypeError
        restore_session(sess, path=CacheSettings.path)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

    except TypeError:
        sess.graph.finalize()
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        train(MAX_STEPS, sess, is_train, train_step, accuracy, loss, VERBOSE)
        save_session(sess, path=CacheSettings.path, sav=saver)

def inference(question,
              sess=sess,
              encoder=corpus_encoder,
              classes=classes,
              query=query,
              embeddings=embeddings,
              limit=5,
              decision_boundary=.85):
    """produce probabilities of most probable topic"""

    encoder, original_pad_length = encoder.fit([question]), encoder.pad_length
    encoded_query = encoder.encode(pad_length=original_pad_length)
    encoded_query = np.array(encoded_query).reshape(-1, original_pad_length)
    embedded_query = encoder.vectorize()

    class_prob = sess.run(fetches=probs,
                          feed_dict={query: encoded_query,
                                     embeddings: embedded_query})[-1].tolist()

    samples = [(class_, class_prob[k]) for k, class_ in enumerate(classes)]

    return feed_conversation(samples, limit, decision_boundary)
<commit_msg>fix queue capacity<commit_after>"""
    lstm (needs modularisation and further refactoring)

    sequence classification of variable lengths through
    Long-short Term Memory (LSTM) which models surrounding context of words.

    Words are represented in 300-dimension vectors to allow generalization over
    semantic redundancy.

    Distributed Representations of Words and Phrases and their Compositionality
    Mikolov et al. 2013
"""
import os

import numpy as np
import tensorflow as tf

from tqdm import tqdm
from datetime import datetime
from sklearn import model_selection, preprocessing

from chatbot.engine import corpus, labels
from chatbot.nlp.embedding import WordEmbedding
from chatbot.nlp.sparse import NLPProcessor
from chatbot.serializers import feed_conversation
from chatbot.settings import (CacheSettings, NLP, FORCE, STATE_SIZE, STEP_SIZE,
                              BATCH_SIZE, MAX_WORDS, MAX_STEPS, VERBOSE)

def resample(docs, labels, sample_size):
    unique_labels = np.unique(labels)
    indice_array = list()
    for label in unique_labels:
        indice_label = np.where(labels == label)[0].tolist()
        indice_array.extend(
            np.random.choice(indice_label, size=sample_size).tolist())
    return (docs[indice_array], labels[indice_array])

def flatten_split_resample(encoded_corpuses, encoded_labels,
                           valid_ratio=.2,
                           sample_size=1000):
    """break documents into sentences and augment, and one-hot encode labels"""

    flattened_docs = list()
    flattened_labels = list()
    for d, l in zip(encoded_corpuses, encoded_labels):
        for sent in d:
            flattened_docs.append(sent)
            flattened_labels.append(l)

    ravelled_corpus = np.array(flattened_docs)
    ravelled_labels = np.array(flattened_labels)

    X_train, X_test, y_train, y_test = \
        model_selection.train_test_split(ravelled_corpus, ravelled_labels,
                                         test_size=valid_ratio,
                                         stratify=ravelled_labels)

    return resample(X_train, y_train, sample_size),\
        resample(X_test, y_test, sample_size)

def size(sequence):
    """produces length vector in shape [batch_size, length] for each padded
    sequence of its true length excluding zero vectors.
    sequence of vectors comes in shape of [batch_size, STEP_SIZE, dimensions]
    """
    flag = tf.sign(tf.reduce_max(tf.abs(sequence), axis=2))
    length = tf.reduce_sum(flag, axis=1)
    return tf.cast(length, tf.int32)

def find_last(outputs, length):
    """zero padded sequences will produce zero vectors after sequence_length
    required using dynamic unrolling dynamic_rnn API. this function locates
    the last non-zero output for calculating logits
    """
    # outputs in shape [BATCH_SIZE, STEP_SIZE, STATE_SIZE], length [BATCH_SIZE]
    b, t, st = tf.unstack(tf.shape(outputs))
    flat_index = tf.range(b) * t + (length - 1)
    zeros = tf.zeros(shape=[b], dtype=tf.int32)
    non_neg_flat_index = tf.maximum(zeros, flat_index)
    flat_outputs = tf.reshape(outputs, [-1, st])
    last_outputs = tf.gather(flat_outputs, non_neg_flat_index)
    return last_outputs

def enqueue(sent_encoded, label_encoded, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued sentence and label pair"""
    sent = tf.convert_to_tensor(sent_encoded, dtype=tf.int32)
    label = tf.convert_to_tensor(label_encoded, dtype=tf.uint8)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[sent, label],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def batch_generator(sent_queue, label_queue, batch_size=None, threads=8):
    return tf.train.shuffle_batch(
                    tensors=[sent_queue, label_queue],
                    batch_size=batch_size,
                    num_threads=threads,
                    capacity=batch_size * 10,
                    min_after_dequeue=batch_size,
                    allow_smaller_final_batch=True)

def restore_session(sess, path):
    """restore hard trained model for predicting."""
    eval_saver = \
        tf.train.import_meta_graph(tf.train.latest_checkpoint(path) + '.meta')
    eval_saver.restore(sess, tf.train.latest_checkpoint(path))
    print('{} restored successfully.'.format(tf.train.latest_checkpoint(path)))

def save_session(sess, path, sav):
    """save hard trained model for future predicting."""
    now = datetime.now().strftime('%Y%m%d%H%M%S')
    if not os.path.exists(path):
        os.makedirs(path)
    save_path = sav.save(sess, path + "model_{0}.ckpt".format(now))
    print("Model saved in: {0}".format(save_path))

def train(n, sess, is_train, optimiser, metric, loss, verbose):

    for global_step in tqdm(range(n), unit='step', disable=verbose):
        _, train_accuracy, train_loss = \
            sess.run([optimiser, metric, loss], feed_dict={is_train: True})

        if verbose:
            print("step {0} of {3}, train accuracy: {1:.4f} log loss: {2:.4f}"
                  .format(global_step, train_accuracy, train_loss, n))

        if global_step and global_step % 100 == 0:
            valid_accuracy, valid_loss = sess.run(fetches=[metric, loss])
            print("step {0} of {3}, valid accuracy: {1:.4f} "
                  "log loss: {2:.4f}".format(global_step, valid_accuracy,
                                             valid_loss, n))

    print("step {0} of {0}, train accuray: {1:.4f}, valid accuracy {2:.4f} "
          "log loss: {3:.4f}".format(n, train_accuracy, valid_accuracy,
                                     train_loss))

corpus = NLPProcessor(attrs=NLP).process(corpus)

corpus_encoder = WordEmbedding(top=MAX_WORDS).fit(corpus)
encoded_corpus = corpus_encoder.encode(zero_pad=True, pad_length=STEP_SIZE)

l_encoder = preprocessing.LabelBinarizer().fit(labels)
encoded_labels, classes = l_encoder.transform(labels), l_encoder.classes_

embedding_matrix = corpus_encoder.vectorize()
embed_shape = embedding_matrix.shape

data = flatten_split_resample(encoded_corpus, encoded_labels, valid_ratio=.2)
train_sent, train_label = batch_generator(*enqueue(*data[0]), BATCH_SIZE)
valid_sent, valid_label = batch_generator(*enqueue(*data[1]), BATCH_SIZE)

query = tf.placeholder_with_default(input=valid_sent,
                                    shape=[None, STEP_SIZE],
                                    name='query')
embeddings = tf.placeholder_with_default(input=embedding_matrix,
                                         shape=[None, embed_shape[-1]],
                                         name='embeddings')
is_train = tf.placeholder_with_default(input=False,
                                       shape=[],
                                       name='is_train_or_valid')

feature_feed = tf.cond(is_train, lambda: train_sent, lambda: query)
label_feed = tf.cond(is_train, lambda: train_label, lambda: valid_label)
keep_prob = tf.cond(is_train, lambda: tf.constant(.8), lambda: tf.constant(1.))

W_softmax = tf.get_variable(name='W',
                            shape=[STATE_SIZE, len(labels)],
                            initializer=tf.truncated_normal_initializer(
                                        stddev=0.1))
b_softmax = tf.get_variable(name='b',
                            shape=[len(labels)],
                            initializer=tf.constant_initializer(0.0))

word_vectors = tf.nn.embedding_lookup(embeddings, feature_feed)

with tf.device('/gpu:0'):

    cell = tf.nn.rnn_cell.BasicLSTMCell(STATE_SIZE)
    cell = tf.nn.rnn_cell.DropoutWrapper(cell=cell,
                                         input_keep_prob=keep_prob,
                                         output_keep_prob=keep_prob,
                                         state_keep_prob=keep_prob)
    sent_length = size(word_vectors)
    outputs, final_state = tf.nn.dynamic_rnn(cell=cell,
                                             inputs=word_vectors,
                                             sequence_length=sent_length,
                                             dtype=tf.float32)

    last = find_last(outputs, sent_length)
    logits = tf.matmul(last, W_softmax) + b_softmax

    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                            labels=label_feed)

    loss = tf.reduce_mean(cross_entropy)
    train_step = tf.train.RMSPropOptimizer(1e-3).minimize(loss)

    probs = tf.nn.softmax(logits)
    correct = tf.equal(tf.argmax(probs, 1), tf.argmax(label_feed, 1))
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

    saver = tf.train.Saver(max_to_keep=5, var_list=tf.global_variables())

    init = tf.global_variables_initializer()

    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
    sess.run(init)

with sess.as_default(), tf.device('/cpu:0'):
    try:
        if FORCE:
            raise TypeError
        restore_session(sess, path=CacheSettings.path)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

    except TypeError:
        sess.graph.finalize()
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        train(MAX_STEPS, sess, is_train, train_step, accuracy, loss, VERBOSE)
        save_session(sess, path=CacheSettings.path, sav=saver)

def inference(question,
              sess=sess,
              encoder=corpus_encoder,
              classes=classes,
              query=query,
              embeddings=embeddings,
              limit=5,
              decision_boundary=.85):
    """produce probabilities of most probable topic"""

    encoder, original_pad_length = encoder.fit([question]), encoder.pad_length
    encoded_query = encoder.encode(pad_length=original_pad_length)
    encoded_query = np.array(encoded_query).reshape(-1, original_pad_length)
    embedded_query = encoder.vectorize()

    class_prob = sess.run(fetches=probs,
                          feed_dict={query: encoded_query,
                                     embeddings: embedded_query})[-1].tolist()

    samples = [(class_, class_prob[k]) for k, class_ in enumerate(classes)]

    return feed_conversation(samples, limit, decision_boundary)
<|endoftext|>


<commit_before>from .chatterbot import ChatBot

__version__ = "0.3.6"
__author__ = "Gunther Cox"
__email__ = "gunthercx@gmail.com"
<commit_msg>Updated version.<commit_after>from .chatterbot import ChatBot

__version__ = "0.3.7"
__author__ = "Gunther Cox"
__email__ = "gunthercx@gmail.com"
<|endoftext|>


<commit_before>from .chatterbot import ChatBot

__version__ = "0.2.4"
__maintainer__ = "Gunther Cox"
__email__ = "gunthercx@gmail.com"
<commit_msg>Updated version.<commit_after>from .chatterbot import ChatBot

__version__ = "0.2.5"
__maintainer__ = "Gunther Cox"
__email__ = "gunthercx@gmail.com"
<|endoftext|>