<commit_before>import psycopg2
import re

from . import dtuple

class ExpressionValue(str):
    pass

def quote_expression_value(value):
    return value

psycopg2.extensions.adapters[ExpressionValue] = quote_expression_value

# Quoting of table/schema names in psycopg2:
# http://osdir.com/ml/python.db.psycopg.devel/2004-10/msg00012.html
class SchemaName(str):
    pass

def quote_schema_name(value):
    return '"' + value.replace('"', '""') + '"'

psycopg2.extensions.adapters[SchemaName] = quote_schema_name

WHITESPACE_REGEXP = re.compile(r'^\s+', re.M)

class CursorWrapper:
    def __init__(self, cursor, conn, debug=False):
        self.cursor = cursor
        self.conn = conn
        self._transaction_depth = 0
        self._transaction_depth_request = 0
        self._debug = debug

    def execute(self, sql, args=None):
        sql = sql.replace('%', '%%').replace('?', '%s')

        try:
            if self._debug:
                debug_sql = WHITESPACE_REGEXP.sub('     ', sql.strip())
                print 'SQL:', debug_sql + ',', repr(args)
            return self.cursor.execute(sql, args)
        except psycopg2.OperationalError, e:
            if str(e).startswith('server closed the connection unexpectedly'):
                if self._transaction_depth == 0:
                    self.conn.reconnect()
                    self.cursor = self.conn.cursor()
                    self.cursor.execute(sql, args)
            else:
                raise

        if self._transaction_depth_request:
            self._transaction_depth += 1
            self._transaction_depth_request -= 1

    def execute_many(self, sql_commands):
        for sql in sql_commands.split(';'):
            sql = sql.strip()
            if sql:
                self.execute(sql)

    #def fetchall(self):
        #return self.cursor.fetchall()

    # Higher-level interface

    def one(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = self.cursor.fetch()
        row = dtuple.DatabaseTuple(desc, row)
        return row

    def one_check(self, sql, args=None):
        row = self.one(sql, args)
        if row is None:
            raise "No data"
        return row

    def all(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [dtuple.DatabaseTuple(desc, row) for row in rows]
        return rows

    def one_value(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = self.cursor.fetch()
        if row is None:
            raise "No data"
        return row[0]

    def all_values(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [row[0] for row in rows]
        return rows

    # Transactions

    def begin(self):
        self._transaction_depth_request += 1

    def commit(self):
        self.conn.commit()
        self._transaction_depth -= 1

    def rollback(self):
        self.conn.rollback()
        self._transaction_depth -= 1

    def close(self):
        return self.cursor.close()

    # Statements

    #def insert_dict(self, table, dict):
    #    table = quote_name(table)
    #    keys_str = ','.join(map(quote_name, dict.keys()))
    #    values = []
    #    values_placeholders= []
    #    for value in dict.values():
    #        if isinstance(value, ExpressionValue):
    #            values_placeholders.append(value)
    #        else:
    #            values_placeholders.append('%s')
    #            values.append(value)
    #    values_str = ','.join(values_placeholders)
    #    sql = 'insert into %s (%s) values (%s)' % (table, keys_str, values_str)
    #    self.execute(sql, values)

    def insert_dict(self, table, dict):
        placeholders = '%s' * len(dict)
        sql = 'insert into %%s (%s) values (%s)' % (placeholders, placeholders)
        table = SchemaName(table)
        columns = [SchemaName(column) for column in dict.keys()]
        values = dict.values()
        self.execute(sql, (table, columns, values))

    # DDL statements

    def add_fkey(self, table, column, target_table=None, target_column=None):
        target_table = column[:-3] + 's'
        target_column = 'id'
        name = '%s_%s_fk' % (table, column)
        vars = {
            'table': table,
            'column': column,
            'name': name,
            'target_table': target_table,
            'target_column': target_column,
        }
        self.execute('''
            alter table %(table)s add constraint %(name)s
                foreign key (%(column)s)
                references %(target_table)s (%(target_column)s)
        ''' % vars)

class CursorContextManager:
    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self.cursor

    def __exit__(self, type, value, traceback):
        self.cursor.close()

class TransactionalCursorContextManager(CursorContextManager):
    def __enter__(self):
        self.cursor.begin()
        return CursorContextManager.__enter__(self)

    def __exit__(self, type, value, traceback):
        if value is None:
            self.cursor.commit()
        else:
            self.cursor.rollback()
        return CursorContextManager.__exit__(self, type, value, traceback)

class ConnectionWrapper:
    def __init__(self, dsn, debug=False):
        self.dsn = dsn
        self.conn = None
        self._debug = debug

    def cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return CursorContextManager(cursor)

    def tx_cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return TransactionalCursorContextManager(cursor)

    # we need to rollback transactions after failed statements
    cursor = tx_cursor

    def commit(self):
        return self.conn.commit()

    def rollback(self):
        return self.conn.rollback()

    def connect(self):
        self.conn = psycopg2.connect(self.dsn)

    def reconnect(self):
        self.connect()

    def expr(self, value):
        return ExpressionValue(self)
<commit_msg>Fixing queries<commit_after>import psycopg2
import re

from . import dtuple

class ExpressionValue(str):
    pass

def quote_expression_value(value):
    return value

psycopg2.extensions.adapters[ExpressionValue] = quote_expression_value

# Quoting of table/schema names in psycopg2:
# http://osdir.com/ml/python.db.psycopg.devel/2004-10/msg00012.html
class SchemaName(str):
    pass

def quote_schema_name(value):
    return '"' + value.replace('"', '""') + '"'

psycopg2.extensions.adapters[SchemaName] = quote_schema_name

WHITESPACE_REGEXP = re.compile(r'^\s+', re.M)

class CursorWrapper:
    def __init__(self, cursor, conn, debug=False):
        self.cursor = cursor
        self.conn = conn
        self._transaction_depth = 0
        self._transaction_depth_request = 0
        self._debug = debug

    def execute(self, sql, args=None):
        sql = sql.replace('?', '%s')

        if args is None:
            args = ()
        elif isinstance(args, basestring) or getattr(args, 'length', None) is None:
            args = (args,)

        try:
            if self._debug:
                debug_sql = WHITESPACE_REGEXP.sub('     ', sql.strip())
                print 'SQL:', debug_sql + ',', repr(args)
            return self.cursor.execute(sql, args)
        except psycopg2.OperationalError, e:
            if str(e).startswith('server closed the connection unexpectedly'):
                if self._transaction_depth == 0:
                    self.conn.reconnect()
                    self.cursor = self.conn.cursor()
                    self.cursor.execute(sql, args)
            else:
                raise

        if self._transaction_depth_request:
            self._transaction_depth += 1
            self._transaction_depth_request -= 1

    def execute_many(self, sql_commands):
        for sql in sql_commands.split(';'):
            sql = sql.strip()
            if sql:
                self.execute(sql)

    #def fetchall(self):
        #return self.cursor.fetchall()

    # Higher-level interface

    def one(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = self.cursor.fetch()
        row = dtuple.DatabaseTuple(desc, row)
        return row

    def one_check(self, sql, args=None):
        row = self.one(sql, args)
        if row is None:
            raise "No data"
        return row

    def all(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [dtuple.DatabaseTuple(desc, row) for row in rows]
        return rows

    def one_value(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = self.cursor.fetch()
        if row is None:
            raise "No data"
        return row[0]

    def all_values(self, sql, args=None):
        self.execute(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [row[0] for row in rows]
        return rows

    # Transactions

    def begin(self):
        self._transaction_depth_request += 1

    def commit(self):
        self.conn.commit()
        self._transaction_depth -= 1

    def rollback(self):
        self.conn.rollback()
        self._transaction_depth -= 1

    def close(self):
        return self.cursor.close()

    # Statements

    #def insert_dict(self, table, dict):
    #    table = quote_name(table)
    #    keys_str = ','.join(map(quote_name, dict.keys()))
    #    values = []
    #    values_placeholders= []
    #    for value in dict.values():
    #        if isinstance(value, ExpressionValue):
    #            values_placeholders.append(value)
    #        else:
    #            values_placeholders.append('%s')
    #            values.append(value)
    #    values_str = ','.join(values_placeholders)
    #    sql = 'insert into %s (%s) values (%s)' % (table, keys_str, values_str)
    #    self.execute(sql, values)

    def insert_dict(self, table, dict):
        placeholders = '%s' * len(dict)
        sql = 'insert into %s (%s) values (%s)' % (placeholders, placeholders)
        table = SchemaName(table)
        columns = [SchemaName(column) for column in dict.keys()]
        values = dict.values()
        self.execute(sql, (table, columns, values))

    # DDL statements

    def add_fkey(self, table, column, target_table=None, target_column=None):
        target_table = column[:-3] + 's'
        target_column = 'id'
        name = '%s_%s_fk' % (table, column)
        vars = {
            'table': table,
            'column': column,
            'name': name,
            'target_table': target_table,
            'target_column': target_column,
        }
        self.execute('''
            alter table %(table)s add constraint %(name)s
                foreign key (%(column)s)
                references %(target_table)s (%(target_column)s)
        ''' % vars)

class CursorContextManager:
    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self.cursor

    def __exit__(self, type, value, traceback):
        self.cursor.close()

class TransactionalCursorContextManager(CursorContextManager):
    def __enter__(self):
        self.cursor.begin()
        return CursorContextManager.__enter__(self)

    def __exit__(self, type, value, traceback):
        if value is None:
            self.cursor.commit()
        else:
            self.cursor.rollback()
        return CursorContextManager.__exit__(self, type, value, traceback)

class ConnectionWrapper:
    def __init__(self, dsn, debug=False):
        self.dsn = dsn
        self.conn = None
        self._debug = debug

    def cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return CursorContextManager(cursor)

    def tx_cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return TransactionalCursorContextManager(cursor)

    # we need to rollback transactions after failed statements
    cursor = tx_cursor

    def commit(self):
        return self.conn.commit()

    def rollback(self):
        return self.conn.rollback()

    def connect(self):
        self.conn = psycopg2.connect(self.dsn)

    def reconnect(self):
        self.connect()

    def expr(self, value):
        return ExpressionValue(self)
<|endoftext|>


<commit_before>#!flask/bin/python

import sys
sys.path.append('./parse_challonge')
from parse_challonge_standings import *
from parse_challonge_info import *
from parse_challonge_matches import *
from parse_challonge_run import *

import re

def main():
  challonge_tournamentlist = sys.argv[1]
  read_tournamentlist(challonge_tournamentlist)

def read_tournamentlist(challonge_tournamentlist):
  # f is a file object
  with open(challonge_tournamentlist, 'r') as f:
    for tournament_line in f:
      # process each line and parse tournament_url, tournament_name
      processed_tournament_line = tournament_line.strip('\n')
      line_parser = re.compile('[|]')
      tokens = line_parser.split(processed_tournament_line)
      tournament_name = tokens[0]
      # Remove '\' escape characters
      tournament_name = tournament_name.translate(None, '\\')
      tournament_url = tokens[1]

      # if optional region parameter is included, run parse_challonge_run with region parameter, otherwise run without it
      if len(tokens)==3:
        tournament_region = tokens[2]
        tournament_region = tournament_region.translate(None, '\\')
        print tournament_name + ': ' + tournament_url
        parse_challonge_run(tournament_url, tournament_name, tournament_region)
      else:
        parse_challonge_run(tournament_url, tournament_name)

  f.close()

if __name__ == "__main__":
  main()
<commit_msg>if no region parameter passed in parse_challonge_tournamentlist, region supplied as None<commit_after>#!flask/bin/python

import sys
sys.path.append('./parse_challonge')
from parse_challonge_standings import *
from parse_challonge_info import *
from parse_challonge_matches import *
from parse_challonge_run import *

import re

def main():
  challonge_tournamentlist = sys.argv[1]
  read_tournamentlist(challonge_tournamentlist)

def read_tournamentlist(challonge_tournamentlist):
  # f is a file object
  with open(challonge_tournamentlist, 'r') as f:
    for tournament_line in f:
      # process each line and parse tournament_url, tournament_name
      processed_tournament_line = tournament_line.strip('\n')
      line_parser = re.compile('[|]')
      tokens = line_parser.split(processed_tournament_line)
      tournament_name = tokens[0]
      # Remove '\' escape characters
      tournament_name = tournament_name.translate(None, '\\')
      tournament_url = tokens[1]

      # if optional region parameter is included, run parse_challonge_run with region parameter, otherwise run without it
      if len(tokens)==3:
        tournament_region = tokens[2]
        tournament_region = tournament_region.translate(None, '\\')
        print tournament_name + ': ' + tournament_url
        parse_challonge_run(tournament_url, tournament_name, tournament_region)
      else:
        parse_challonge_run(tournament_url, tournament_name, None)

  f.close()

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>import transaction
from zExceptions import Redirect
from zope.component import getMultiAdapter
from zope.event import notify
from zope.i18n import translate
from zope.interface import implementer
from yafowil.base import ExtractionError
from yafowil.base import UNSET
from yafowil.base import factory
from yafowil.plone.form import Form
from yafowil.yaml import parse_from_YAML
from bda.plone.cart import readcookie
from bda.plone.cart import get_data_provider
from bda.plone.checkout import CheckoutDone
from bda.plone.checkout import message_factory as _
from bda.plone.checkout.interfaces import CheckoutError
from bda.plone.checkout.interfaces import ICheckoutAdapter
from bda.plone.checkout.interfaces import ICheckoutSettings
from bda.plone.checkout.interfaces import ICheckoutFormPresets
from bda.plone.checkout.interfaces import IFieldsProvider
from bda.plone.checkout.vocabularies import country_vocabulary
from bda.plone.checkout.vocabularies import gender_vocabulary
from bda.plone.payment import Payments
from bda.plone.shipping import Shippings
import plone.api

TERMS_AND_CONDITONS_ID = 'agb'

class ProviderRegistry(object):

    def __init__(self):
        self.providers = list()

    def add(self, factory):
        self.providers.append(factory)

    def __iter__(self):
        return self.providers.__iter__()

provider_registry = ProviderRegistry()

CHECKOUT = 0
CONFIRM = 1

class FormContext(object):

    @property
    def form_context(self):
        confirm = self.request.get('checkout_confirm') or\
            self.request.get('action.checkout.finish')
        return confirm and CONFIRM or CHECKOUT

    @property
    def mode(self):
        return self.form_context is CONFIRM and 'display' or 'edit'

@implementer(IFieldsProvider)
class FieldsProvider(FormContext):
    fields_template = None
    fields_name = ''
    message_factory = _
    ignore_on_save = False
    skip = False

    def __init__(self, context, request):
        self.context = context
        self.request = request
        self.preset_adapter = getMultiAdapter(
            (self.context, self.request),
            ICheckoutFormPresets
        )

    def get_value(self, widget, data):
        """Function to fetch form field default values.

        Special case in checkout form is that form fields might get hidden mode
        """
        default = self.preset_adapter.get_value(widget.dottedpath)
        ret = None
        # XXX: improve, looks odd
        if 'checkbox' in widget.blueprints:
            # for selected checkboxes, not the value but only the input name
            # (dottedpath) is in request.form, otherwise they are completly
            # ommited.
            ret = widget.dottedpath in self.request or default
        else:
            ret = self.request.get(widget.dottedpath, default)
        return ret

    def extend(self, form):
        if self.skip:
            return
        fields = parse_from_YAML(self.fields_template,
                                 self, self.message_factory)
        form[self.fields_name] = fields

class CartSummary(FieldsProvider):
    fields_name = 'cart_summary'

    def extend(self, form):
        if not self.form_context == CONFIRM:
            return
        compound = form[self.fields_name] = factory('compound', props={
            'structural': True})
        compound['heading'] = factory('tag', props={
            'structural': True,
            'tag': 'h2',
            'text': _('heading_cart_summary', 'Cart')})
        compound['overview'] = factory('tag', props={
            'structural': True,
            'class': 'cart_overview',
            'tag': 'div',
            'text': self.context.restrictedTraverse('@@cart_overview')()})

provider_registry.add(CartSummary)

class PersonalData(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/personal_data.yaml'
    fields_name = 'personal_data'

    @property
    def gender_vocabulary(self):
        return [('-', '')] + gender_vocabulary()

provider_registry.add(PersonalData)

class BillingAddress(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/billing_address.yaml'
    fields_name = 'billing_address'

    @property
    def country_vocabulary(self):
        return country_vocabulary()

provider_registry.add(BillingAddress)

class DeliveryAddress(BillingAddress):
    fields_template = 'bda.plone.checkout.browser:forms/delivery_address.yaml'
    fields_name = 'delivery_address'

    def conditional_required(self, widget, data):
        if data.parent['alternative_delivery'].extracted\
                and not data.extracted:
            raise ExtractionError(widget.attrs['conditional_required'])
        return data.extracted

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.delivery_address.alternative_delivery'
        if self.request.get(name) is None:
            return 'hidden'
        return ''

    @property
    def alternative_delivery_vocab(self):
        return {
            True: _('yes', 'Yes'),
            False: _('no', 'No'),
            UNSET: _('not set', 'not set'),
        }

provider_registry.add(DeliveryAddress)

class ShippingSelection(FieldsProvider):
    fields_template =\
        'bda.plone.checkout.browser:forms/shipping_selection.yaml'
    fields_name = 'shipping_selection'

    @property
    def skip(self):
        cart_data = get_data_provider(self.context, self.request)
        return not cart_data.include_shipping_costs

    @property
    def shippings(self):
        return Shippings(self.context)

    @property
    def shipping_vocabulary(self):
        vocab = list()
        for sh in self.shippings.shippings:
            if not sh.available:
                continue
            if sh.description:
                label = translate(sh.label, context=self.request)
                desc = translate(sh.description, context=self.request)
                title = '%s (%s)' % (label, desc)
            else:
                title = translate(sh.label, context=self.request)
            vocab.append((sh.sid, title))
        return vocab

    def get_shipping(self, widget, data):
        default_shipping = self.shippings.default
        request = self.request
        from_request = request.get(widget.dottedpath)
        from_cookie = request.cookies.get('shipping_method')
        # got selection from request which differs from cookie, set cookie
        if from_request and from_cookie != from_request:
            request.response.setCookie(
                'shipping_method', from_request, quoted=False, path='/')
        # no shipping from request or cookie, return default
        if not from_request and not from_cookie:
            return self.shippings.default
        # shipping from cookie, but not from request, return cookie
        if from_cookie and not from_request:
            return from_cookie
        # return from request
        return from_request

provider_registry.add(ShippingSelection)

class PaymentSelection(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/payment_selection.yaml'
    fields_name = 'payment_selection'

    @property
    def skip(self):
        cart_data = get_data_provider(self.context, self.request)
        return not cart_data.total

    @property
    def payments(self):
        return Payments(self.context)

    @property
    def payment_vocabulary(self):
        return self.payments.vocab

    def get_payment(self, widget, data):
        return self.request.get(widget.dottedpath, self.payments.default)

provider_registry.add(PaymentSelection)

class OrderComment(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/order_comment.yaml'
    fields_name = 'order_comment'

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.order_comment.comment'
        if not self.request.get(name):
            return 'hidden'
        return ''

provider_registry.add(OrderComment)

class AcceptTermsAndConditions(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/accept_terms.yaml'
    fields_name = 'accept_terms_and_conditions'
    ignore_on_save = True

    @property
    def accept_label(self):
        base = plone.api.portal.get_navigation_root(self.context).absolute_url()
        # XXX: url from config
        tac_url = '%s/%s' % (base, TERMS_AND_CONDITONS_ID)
        tac_label = _('terms_and_conditions', 'Terms and conditions')
        tac_label = translate(tac_label, context=self.request)
        tac_link = '%s'
        tac_link = tac_link % (tac_url, tac_label)
        tac_text = _('terms_and_conditions_text',
                     'I have read and accept the ${terms_and_conditions}',
                     mapping={'terms_and_conditions': tac_link})
        return tac_text

    @property
    def mode(self):
        if self.request.get('action.checkout.finish')\
                or self.form_context is CONFIRM:
            return 'edit'
        return 'skip'

    def validate_accept(self, widget, data):
        if not data.extracted:
            raise ExtractionError(_('error_accept_terms_and_conditiond',
                                    'Please accept our terms and conditions.'))
        return data.extracted

provider_registry.add(AcceptTermsAndConditions)

class CheckoutForm(Form, FormContext):
    action_resource = '@@checkout'
    # in order to provide your own registry, subclass CheckoutForm and and
    # override ``provider_registry``
    provider_registry = provider_registry

    def prepare(self):
        if not readcookie(self.request):
            raise Redirect(self.context.absolute_url())
        checkout = self.form_context is CHECKOUT
        form_class = checkout and 'mode_edit' or 'mode_display'
        self.form = factory('#form', name='checkout', props={
            'action': self.form_action,
            'class_add': form_class})
        for fields_factory in self.provider_registry:
            fields_factory(self.context, self.request).extend(self.form)
        # checkout data input
        if checkout:
            self.form['checkout_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'checkout_back',
                'handler': None,
                'next': self.checkout_back,
                'skip': True})
            self.form['next'] = factory('submit', props={
                'label': _('next', 'Next'),
                'action': 'next',
                'handler': None,
                'next': self.checkout_summary})
        # checkout confirmation
        else:
            self.form['confirm_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'confirm_back',
                'handler': None,
                'next': self.confirm_back})
            self.form['finish'] = factory('submit', props={
                'class': 'prevent_if_no_longer_available',
                'label': _('finish', 'Order now'),
                'action': 'finish',
                'handler': self.finish,
                'next': self.checkout_done})

    def checkout_back(self, request):
        raise Redirect('%s/@@cart' % self.context.absolute_url())

    def confirm_back(self, request):
        self.prepare()
        return self.form(request=request)

    def checkout_summary(self, request):
        self.request['checkout_confirm'] = '1'
        self.prepare()
        return self.form(request=request)

    def checkout_done(self, request):
        transaction.commit()
        raise Redirect(self.finish_redirect_url)

    def finish(self, widget, data):
        providers = [fields_factory(self.context, self.request)
                     for fields_factory in self.provider_registry]
        to_adapt = (self.context, self.request)
        checkout_adapter = getMultiAdapter(to_adapt, ICheckoutAdapter)
        try:
            uid = checkout_adapter.save(providers, widget, data)
        except CheckoutError:
            transaction.abort()
            self.checkout_back(self.request)
        checkout_adapter.clear_session()
        checkout_settings = ICheckoutSettings(self.context)
        if checkout_settings.skip_payment(uid):
            self.finish_redirect_url = \
                checkout_settings.skip_payment_redirect_url(uid)
        else:
            p_name = data.fetch('checkout.payment_selection.payment').extracted
            payments = Payments(self.context)
            payment = payments.get(p_name)
            self.finish_redirect_url = payment.init_url(str(uid))
        event = CheckoutDone(self.context, self.request, uid)
        notify(event)
<commit_msg>removed superfluos assignment, some pep8<commit_after>import transaction
from zExceptions import Redirect
from zope.component import getMultiAdapter
from zope.event import notify
from zope.i18n import translate
from zope.interface import implementer
from yafowil.base import ExtractionError
from yafowil.base import UNSET
from yafowil.base import factory
from yafowil.plone.form import Form
from yafowil.yaml import parse_from_YAML
from bda.plone.cart import readcookie
from bda.plone.cart import get_data_provider
from bda.plone.checkout import CheckoutDone
from bda.plone.checkout import message_factory as _
from bda.plone.checkout.interfaces import CheckoutError
from bda.plone.checkout.interfaces import ICheckoutAdapter
from bda.plone.checkout.interfaces import ICheckoutSettings
from bda.plone.checkout.interfaces import ICheckoutFormPresets
from bda.plone.checkout.interfaces import IFieldsProvider
from bda.plone.checkout.vocabularies import country_vocabulary
from bda.plone.checkout.vocabularies import gender_vocabulary
from bda.plone.payment import Payments
from bda.plone.shipping import Shippings
import plone.api

TERMS_AND_CONDITONS_ID = 'agb'

class ProviderRegistry(object):

    def __init__(self):
        self.providers = list()

    def add(self, factory):
        self.providers.append(factory)

    def __iter__(self):
        return self.providers.__iter__()

provider_registry = ProviderRegistry()

CHECKOUT = 0
CONFIRM = 1

class FormContext(object):

    @property
    def form_context(self):
        confirm = self.request.get('checkout_confirm') or\
            self.request.get('action.checkout.finish')
        return confirm and CONFIRM or CHECKOUT

    @property
    def mode(self):
        return self.form_context is CONFIRM and 'display' or 'edit'

@implementer(IFieldsProvider)
class FieldsProvider(FormContext):
    fields_template = None
    fields_name = ''
    message_factory = _
    ignore_on_save = False
    skip = False

    def __init__(self, context, request):
        self.context = context
        self.request = request
        self.preset_adapter = getMultiAdapter(
            (self.context, self.request),
            ICheckoutFormPresets
        )

    def get_value(self, widget, data):
        """Function to fetch form field default values.

        Special case in checkout form is that form fields might get hidden mode
        """
        default = self.preset_adapter.get_value(widget.dottedpath)
        ret = None
        # XXX: improve, looks odd
        if 'checkbox' in widget.blueprints:
            # for selected checkboxes, not the value but only the input name
            # (dottedpath) is in request.form, otherwise they are completly
            # ommited.
            ret = widget.dottedpath in self.request or default
        else:
            ret = self.request.get(widget.dottedpath, default)
        return ret

    def extend(self, form):
        if self.skip:
            return
        fields = parse_from_YAML(self.fields_template,
                                 self, self.message_factory)
        form[self.fields_name] = fields

class CartSummary(FieldsProvider):
    fields_name = 'cart_summary'

    def extend(self, form):
        if not self.form_context == CONFIRM:
            return
        compound = form[self.fields_name] = factory('compound', props={
            'structural': True})
        compound['heading'] = factory('tag', props={
            'structural': True,
            'tag': 'h2',
            'text': _('heading_cart_summary', 'Cart')})
        compound['overview'] = factory('tag', props={
            'structural': True,
            'class': 'cart_overview',
            'tag': 'div',
            'text': self.context.restrictedTraverse('@@cart_overview')()})

provider_registry.add(CartSummary)

class PersonalData(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/personal_data.yaml'
    fields_name = 'personal_data'

    @property
    def gender_vocabulary(self):
        return [('-', '')] + gender_vocabulary()

provider_registry.add(PersonalData)

class BillingAddress(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/billing_address.yaml'
    fields_name = 'billing_address'

    @property
    def country_vocabulary(self):
        return country_vocabulary()

provider_registry.add(BillingAddress)

class DeliveryAddress(BillingAddress):
    fields_template = 'bda.plone.checkout.browser:forms/delivery_address.yaml'
    fields_name = 'delivery_address'

    def conditional_required(self, widget, data):
        if data.parent['alternative_delivery'].extracted\
                and not data.extracted:
            raise ExtractionError(widget.attrs['conditional_required'])
        return data.extracted

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.delivery_address.alternative_delivery'
        if self.request.get(name) is None:
            return 'hidden'
        return ''

    @property
    def alternative_delivery_vocab(self):
        return {
            True: _('yes', 'Yes'),
            False: _('no', 'No'),
            UNSET: _('not set', 'not set'),
        }

provider_registry.add(DeliveryAddress)

class ShippingSelection(FieldsProvider):
    fields_template =\
        'bda.plone.checkout.browser:forms/shipping_selection.yaml'
    fields_name = 'shipping_selection'

    @property
    def skip(self):
        cart_data = get_data_provider(self.context, self.request)
        return not cart_data.include_shipping_costs

    @property
    def shippings(self):
        return Shippings(self.context)

    @property
    def shipping_vocabulary(self):
        vocab = list()
        for sh in self.shippings.shippings:
            if not sh.available:
                continue
            if sh.description:
                label = translate(sh.label, context=self.request)
                desc = translate(sh.description, context=self.request)
                title = '%s (%s)' % (label, desc)
            else:
                title = translate(sh.label, context=self.request)
            vocab.append((sh.sid, title))
        return vocab

    def get_shipping(self, widget, data):
        request = self.request
        from_request = request.get(widget.dottedpath)
        from_cookie = request.cookies.get('shipping_method')
        # got selection from request which differs from cookie, set cookie
        if from_request and from_cookie != from_request:
            request.response.setCookie(
                'shipping_method', from_request, quoted=False, path='/')
        # no shipping from request or cookie, return default
        if not from_request and not from_cookie:
            return self.shippings.default
        # shipping from cookie, but not from request, return cookie
        if from_cookie and not from_request:
            return from_cookie
        # return from request
        return from_request

provider_registry.add(ShippingSelection)

class PaymentSelection(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/payment_selection.yaml'
    fields_name = 'payment_selection'

    @property
    def skip(self):
        cart_data = get_data_provider(self.context, self.request)
        return not cart_data.total

    @property
    def payments(self):
        return Payments(self.context)

    @property
    def payment_vocabulary(self):
        return self.payments.vocab

    def get_payment(self, widget, data):
        return self.request.get(widget.dottedpath, self.payments.default)

provider_registry.add(PaymentSelection)

class OrderComment(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/order_comment.yaml'
    fields_name = 'order_comment'

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.order_comment.comment'
        if not self.request.get(name):
            return 'hidden'
        return ''

provider_registry.add(OrderComment)

class AcceptTermsAndConditions(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/accept_terms.yaml'
    fields_name = 'accept_terms_and_conditions'
    ignore_on_save = True

    @property
    def accept_label(self):
        nav_root = plone.api.portal.get_navigation_root(self.context)
        base = nav_root.absolute_url()
        # XXX: url from config
        tac_url = '%s/%s' % (base, TERMS_AND_CONDITONS_ID)
        tac_label = _('terms_and_conditions', 'Terms and conditions')
        tac_label = translate(tac_label, context=self.request)
        tac_link = '%s'
        tac_link = tac_link % (tac_url, tac_label)
        tac_text = _('terms_and_conditions_text',
                     'I have read and accept the ${terms_and_conditions}',
                     mapping={'terms_and_conditions': tac_link})
        return tac_text

    @property
    def mode(self):
        if self.request.get('action.checkout.finish')\
                or self.form_context is CONFIRM:
            return 'edit'
        return 'skip'

    def validate_accept(self, widget, data):
        if not data.extracted:
            raise ExtractionError(_('error_accept_terms_and_conditiond',
                                    'Please accept our terms and conditions.'))
        return data.extracted

provider_registry.add(AcceptTermsAndConditions)

class CheckoutForm(Form, FormContext):
    action_resource = '@@checkout'
    # in order to provide your own registry, subclass CheckoutForm and and
    # override ``provider_registry``
    provider_registry = provider_registry

    def prepare(self):
        if not readcookie(self.request):
            raise Redirect(self.context.absolute_url())
        checkout = self.form_context is CHECKOUT
        form_class = checkout and 'mode_edit' or 'mode_display'
        self.form = factory('#form', name='checkout', props={
            'action': self.form_action,
            'class_add': form_class})
        for fields_factory in self.provider_registry:
            fields_factory(self.context, self.request).extend(self.form)
        # checkout data input
        if checkout:
            self.form['checkout_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'checkout_back',
                'handler': None,
                'next': self.checkout_back,
                'skip': True})
            self.form['next'] = factory('submit', props={
                'label': _('next', 'Next'),
                'action': 'next',
                'handler': None,
                'next': self.checkout_summary})
        # checkout confirmation
        else:
            self.form['confirm_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'confirm_back',
                'handler': None,
                'next': self.confirm_back})
            self.form['finish'] = factory('submit', props={
                'class': 'prevent_if_no_longer_available',
                'label': _('finish', 'Order now'),
                'action': 'finish',
                'handler': self.finish,
                'next': self.checkout_done})

    def checkout_back(self, request):
        raise Redirect('%s/@@cart' % self.context.absolute_url())

    def confirm_back(self, request):
        self.prepare()
        return self.form(request=request)

    def checkout_summary(self, request):
        self.request['checkout_confirm'] = '1'
        self.prepare()
        return self.form(request=request)

    def checkout_done(self, request):
        transaction.commit()
        raise Redirect(self.finish_redirect_url)

    def finish(self, widget, data):
        providers = [fields_factory(self.context, self.request)
                     for fields_factory in self.provider_registry]
        to_adapt = (self.context, self.request)
        checkout_adapter = getMultiAdapter(to_adapt, ICheckoutAdapter)
        try:
            uid = checkout_adapter.save(providers, widget, data)
        except CheckoutError:
            transaction.abort()
            self.checkout_back(self.request)
        checkout_adapter.clear_session()
        checkout_settings = ICheckoutSettings(self.context)
        if checkout_settings.skip_payment(uid):
            self.finish_redirect_url = \
                checkout_settings.skip_payment_redirect_url(uid)
        else:
            p_name = data.fetch('checkout.payment_selection.payment').extracted
            payments = Payments(self.context)
            payment = payments.get(p_name)
            self.finish_redirect_url = payment.init_url(str(uid))
        event = CheckoutDone(self.context, self.request, uid)
        notify(event)
<|endoftext|>


<commit_before>"""the core components of the ORM: sessions, resources, and fields"""
import collections
import types

class Session(object):
    """the context in which resources are used"""

    def __init__(self):
        for name, resource_class in self.resources.items():
            klass = types.new_class(name, bases=(resource_class, ),
                                    kwds={'session': self})
            setattr(self, name, klass)

    def __init_subclass__(cls, **kwargs):
        cls.resources: Mapping[str, type] = {}
        super().__init_subclass__(**kwargs)

    @classmethod
    def register_resource(cls, resource_cls: type) -> None:
        cls.resources[resource_cls.__name__] = resource_cls

class Resource:
    """base class for API resources"""

    def __init_subclass__(cls, session_cls: type=None,
                          session: Session=None, **kwargs):
        """initialize a Resource subclass

        Parameters
        ----------
        session_cls
            the :class:`Session` subclass to bind to this resource
        session
            the :class:`Session` instance to bind this resource
        """
        if session_cls:
            session_cls.register_resource(cls)
        elif cls.__bases__ == (Resource, ):
            raise Exception(
                'subclassing ``Resource`` requires a session class')

        if session:
            cls.session = session

        cls.fields = collections.OrderedDict(
            (name, obj) for name, obj in cls.__dict__.items()
            if isinstance(obj, Field)
        )
        super().__init_subclass__(**kwargs)

    @classmethod
    def wrap_api_obj(cls, api_obj: object) -> 'Resource':
        """wrap the API object in a resource instance

        Parameters
        ----------
        api_obj
            the API object to wrap

        Returns
        -------
        Resource
            the resource instance
        """
        instance = cls.__new__(cls)
        instance.api_obj = api_obj
        return instance

class Field(object):
    """an attribute accessor for a resource"""

    def __set_name__(self, resource, name):
        self.resource, self.name = resource, name

    def __get__(self, instance, cls):
        if instance is not None:
            return self.get_value(instance)
        else:
            return self

    def get_value(self, instance: Resource) -> object:
        raise NotImplementedError()
<commit_msg>small refactor<commit_after>"""the core components of the ORM: sessions, resources, and fields"""
import collections
import types

class Session(object):
    """the context in which resources are used"""

    def __init__(self):
        for name, resource_class in self.resources.items():
            klass = types.new_class(name, bases=(resource_class, ),
                                    kwds={'session': self})
            setattr(self, name, klass)

    def __init_subclass__(cls, **kwargs):
        cls.resources: Mapping[str, type] = {}
        super().__init_subclass__(**kwargs)

    @classmethod
    def register_resource(cls, resource_cls: type) -> None:
        cls.resources[resource_cls.__name__] = resource_cls

class Resource:
    """base class for API resources"""

    def __init_subclass__(cls, session_cls: type=None,
                          session: Session=None, **kwargs):
        """initialize a Resource subclass

        Parameters
        ----------
        session_cls
            the :class:`Session` subclass to bind to this resource
        session
            the :class:`Session` instance to bind this resource
        """
        if session_cls:
            session_cls.register_resource(cls)
        elif cls.__bases__ == (Resource, ):
            raise Exception(
                'subclassing ``Resource`` requires a session class')

        if session:
            cls.session = session

        cls.fields = collections.OrderedDict(
            (name, obj) for name, obj in cls.__dict__.items()
            if isinstance(obj, Field)
        )
        super().__init_subclass__(**kwargs)

    @classmethod
    def wrap_api_obj(cls, api_obj: object) -> 'Resource':
        """wrap the API object in a resource instance

        Parameters
        ----------
        api_obj
            the API object to wrap

        Returns
        -------
        Resource
            the resource instance
        """
        instance = cls.__new__(cls)
        instance.api_obj = api_obj
        return instance

class Field:
    """an attribute accessor for a resource"""

    def __set_name__(self, resource, name):
        self.resource, self.name = resource, name

    def __get__(self, instance, cls):
        if instance is not None:
            return self.get_value(instance)
        else:
            return self

    def get_value(self, instance: Resource) -> object:
        raise NotImplementedError()
<|endoftext|>


<commit_before>import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
from parsl.monitoring.web_app.app import app, init_db
import sys, getopt

def web_app(db, port):
    if not init_db(db):
        return

    from parsl.monitoring.web_app.apps import sql, workflows

    app.layout = html.Div([
        # html.Link(href='/assets/styles.css', rel='stylesheet'),
        dcc.Location(id='url', refresh=False),
        html.Nav(children=[
            dcc.Link('Workflows', href='/workflows'),
            dcc.Link('SQL', href='/sql')]),
        html.Div(id='page-content')
    ])

    @app.callback(Output('page-content', 'children'),
                  [Input('url', 'pathname')])
    def display_page(pathname):
        if pathname == "/":
            dcc.Location(id='url', refresh=True, pathname="/workflows"),
        elif pathname == "/workflows":
            return workflows.layout
        elif pathname == "/sql":
            return sql.layout
        else:
            return '404'

    app.run_server(port=port, debug=True, use_reloader=False)

# TODO CSS is not being imported when ran through cli
def cli_run():
    argv = sys.argv[1:]
    db_name = 'parsl.db'
    db_dir = './'
    port = 8050
    try:
        opts, args = getopt.getopt(argv,"h:o",["db_dir=","db_name=","port="])
    except getopt.GetoptError:
        print('Invalid argument')
        print('parsl-visualize --db_dir <db_dir> --db_name <db_name> --port <port>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('parsl-visualize --db_dir <db_dir> --db_name <db_name> --port <port>')
            sys.exit()
        elif opt in ("--db_dir", ):
            db_dir = arg
        elif opt in ("--db_name", ):
            db_name = arg
        elif opt in ("--port", ):
            if not arg.isdigit():
                print('Port number must be an integer')
                sys.exit(2)
            port = arg

    print('db_name =', db_name)
    print('db_dir =', db_dir)
    print('port =', port)

    web_app(db_dir + db_name, port)

def run(monitoring_config):
    db = monitoring_config.eng_link.split('/').pop()
    port = monitoring_config.web_app_port + 1
    web_app(db, port)

#
# if __name__ == '__main__':
#     cli_run(argv = sys.argv[1:])
<commit_msg>Polished up cli_run and display_page functions<commit_after>import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
from parsl.monitoring.web_app.app import app, init_db
import sys
import getopt

def web_app(db, port):
    if not init_db(db):
        return

    from parsl.monitoring.web_app.apps import sql, workflows, workflow_details

    app.layout = html.Div([
        dcc.Location(id='url', refresh=False),
        html.Nav(children=[
            dcc.Link('Workflows', href='/workflows'),
            dcc.Link('SQL', href='/sql')]),
        html.Div(id='page-content')
    ])

    @app.callback(Output('page-content', 'children'),
                  [Input('url', 'pathname')])
    def display_page(pathname):
        if pathname == '/': # TODO Redirect to workflows or show special page
            pass
        elif pathname == '/workflows':
            return workflows.layout
        elif '/workflows' in str(pathname):
            return workflow_details.display_workflow(run_id=pathname.split('/').pop())
        elif pathname == '/sql':
            return sql.layout
        else:
            return '404'

    # Setting use_reloader to false prevents the script to be ran twice
    app.run_server(port=port, debug=True, use_reloader=False)

# TODO Automatically searching for .db files would be a nice touch
def cli_run():
    argv = sys.argv[1:]
    db_name = 'parsl.db'
    db_dir = './'
    port = 8050
    try:
        opts, args = getopt.getopt(argv,'h',['db_dir=','db_name=','port='])
    except getopt.GetoptError:
        print('Invalid argument')
        print('parsl-visualize --db_dir <db_dir> --db_name <db_name> --port <port>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('parsl-visualize --db_dir <db_dir> --db_name <db_name> --port <port>')
            sys.exit()
        elif opt in ('--db_dir', ):
            db_dir = arg
        elif opt in ('--db_name', ):
            db_name = arg
        elif opt in ('--port', ):
            if not arg.isdigit():
                print('Port number must be an integer')
                sys.exit(2)
            port = arg

    print('db_name =', db_name)
    print('db_dir =', db_dir)
    print('port =', port)

    web_app(db_dir + db_name, port)

def run(monitoring_config):
    db = monitoring_config.eng_link.split('/').pop()
    port = monitoring_config.web_app_port + 1

    print('db =', db)
    print('Visualization port =', port)

    web_app(db, port)

if __name__ == '__main__':
    cli_run()
<|endoftext|>


<commit_before>import pycountry
from Products.CMFPlone.utils import safe_unicode
from bda.plone.checkout import message_factory as _
from zope.i18nmessageid import MessageFactory

_c = MessageFactory('iso3166')

def gender_vocabulary():
    return [('male', _('male', default=u'Male')),
            ('female', _('female', default=u'Female'))]

AVAILABLE_COUNTRIES = [
    (country.numeric, safe_unicode(_c(country.name))) \
        for country in pycountry.countries
]
# patch this list to modify available countries
ENABLED_COUNTRIES = [
    '040', # Austria
    '756', # Switzerland
    '276', # Germany
    '380', # Italy
]

def country_vocabulary():
    """Vocabulary for countries from ISO3166 source.
    """
    ret = list()
    for cid, name in AVAILABLE_COUNTRIES:
        if cid in ENABLED_COUNTRIES:
            ret.append((cid, name))
    return ret

def get_pycountry_name(country_id):
    if not country_id:
        return None
    country = pycountry.countries.get(numeric=country_id)
    return _c(country.name)
<commit_msg>update ENABLED_COUNTRIES per sites and refactor code slightly<commit_after>import pycountry
from Products.CMFPlone.utils import safe_unicode
from bda.plone.checkout import message_factory as _
from zope.i18nmessageid import MessageFactory
from plone import api

_c = MessageFactory('iso3166')

def gender_vocabulary():
    return [('male', _('male', default=u'Male')),
            ('female', _('female', default=u'Female'))]

AVAILABLE_COUNTRIES = [
    (country.numeric, safe_unicode(_c(country.name)))
    for country in pycountry.countries
]
# patch this list to modify available countries

ENABLED_COUNTRIES = {
    'ElectronzNZ': [
        '554',  # New Zealand
        ],
    'GardeningSolutionzTM': [
        '554',  # New Zealand
        ],
    'GirlGuidingNZ': [
        '554',  # New Zealand
        ],
    'JKKKayaks': [
        '554',  # New Zealand
        ],
    'default': [
        '826',  # United Kingdom
        '554',  # New Zealand
        '040',  # Austria
        '756',  # Switzerland
        '276',  # Germany
        '380',  # Italy
        ],
    }

def country_vocabulary():
    """Vocabulary for countries from ISO3166 source.
    """
    site_id = api.portal.get().getId()

    ret = list()
    for cid, name in AVAILABLE_COUNTRIES:
        if cid in ENABLED_COUNTRIES.get(site_id, ENABLED_COUNTRIES['default']):
            ret.append((cid, name))
    return ret

def get_pycountry_name(country_id):
    if not country_id:
        return None
    country = pycountry.countries.get(numeric=country_id)
    return _c(country.name)
<|endoftext|>


<commit_before>import re
import urllib2
import sqlite3
from bs4 import BeautifulSoup
from urlparse import urljoin
from urlparse import urlparse

ignoreWords=set(['the','of','to','and','a','in','is','it'])

class crawler:
    # Initialize the crawler with the name of database
    def __init__(self,dbName):
        self.con=sqlite3.connect(dbName)

    def __del__(self):
        self.con.close()

    def dbCommit(self):
        self.con.commit()

    # Auxilliary function for getting an entry id and adding
    # it if it's not present
    def getEntryId(self,table,field,value,createNew=True):
        cur=self.con.execute("select rowid from %s where %s='%s'" % (table,field,value))
        res=cur.fetchone()
        if res==None:
            cur=self.con.execute("insert into %s(%s) values('%s')" % (table,field,value))
            return cur.lastrowid
        return res[0]

    # Index an individual page
    def addToIndex(self,url,soup):
        if self.isIndexed(url):
            return
        print 'Indexing %s' % url
        urlid=self.getEntryId('urllist','url',url)

        text=self.getTextOnly(soup)
        words=self.separateWords(text)
        for i in range(len(words)):
            word=words[i]
            if word in ignoreWords:
                continue
            wordid=self.getEntryId('wordlist','word',word)
            self.con.execute('insert into wordlocation(urlid,wordid,location) values(%d,%d,%d)' % (urlid,wordid,i))

    # Extract the text from an HTML page (no tags)
    def getTextOnly(self,soup):
        v=soup.string
        if v == None:
            result=''
            for content in soup.contents:
                result+=self.getTextOnly(content)+'\n'
            return result
        return v.strip()

    # Separate the words by any non-whitespace character
    def separateWords(self,text):
        splitter=re.compile('[^a-zA-Z0-9_+-]')
        return [t for t in splitter.split(text) if t!='']

    # Return true if this url is already indexed
    def isIndexed(self,url):
        res=self.con.execute("select rowid from urllist where url='%s'" % url).fetchone()
        if res!=None:
            urlid=res[0]
            res=self.con.execute("select rowid from wordlocation where urlid=%d" % urlid).fetchone()
            return res!=None
        return False

    # Add a link between two pages
    def addLinkRef(self,urlFrom,urlTo,linkText):
        pass

    # Starting with a list of pages, do a breadth
    # first search to the given depth, indexing pages
    # as we go
    def crawl(self,pages,depth=2):
        for i in range(depth):
            newPages=set()
            for page in pages:
                try:
                    c=urllib2.urlopen(page)
                except:
                    print "Could not open %s" % page

                soup=BeautifulSoup(c, 'html.parser')
                self.addToIndex(page,soup)
                parsedUri=urlparse(page)
                scheme='{uri.scheme}'.format(uri=parsedUri)
                domain='{uri.scheme}://{uri.netloc}/'.format(uri=parsedUri)
                links=soup('a')
                for link in links:
                    if 'href' in dict(link.attrs):
                        uri=urljoin(domain,link['href'])
                        if uri.find("'")!=-1: continue
                        uri=uri.split('#')[0]
                        if (scheme == 'http' or scheme == 'https') and not self.isIndexed(uri):
                            newPages.add(uri)
                        linkText=self.getTextOnly(link)
                        self.addLinkRef(page,uri,linkText)
                self.dbCommit()

            pages=newPages

    # Create the database tables
    def createIndexTables(self):
        self.con.execute('create table urllist(url)')
        self.con.execute('create table wordlist(word)')
        self.con.execute('create table wordlocation(urlid,wordid,location)')
        self.con.execute('create table link(fromid,toid)')
        self.con.execute('create table linkwords(wordid,linkid)')
        self.con.execute('create index urlidx on urllist(url)')
        self.con.execute('create index wordidx on wordlist(word)')
        self.con.execute('create index wordurlidx on wordlocation(wordid)')
        self.con.execute('create index urltoidx on link(toid)')
        self.con.execute('create index urlfromidx on link(fromid)')
        self.dbCommit()

class searcher:
    def __init__(self,dbName):
        self.con=sqlite3.connect(dbName)

    def __del__(self):
        self.con.close()

    def getMatchRows(self,q):
        fieldlist = 'w0.urlid'
        tablelist = ''
        clauselist = ''
        wordids = []

        words = q.split(' ')
        tableNumber = 0

        for word in words:
            res=self.con.execute("select rowid from wordlist where word='%s'" % word).fetchone()
            if res!=None:
                wordid = res[0]
                wordids.append(wordid)

                if tableNumber > 0:
                    clauselist += ' and w%d.urlid=w%d.urlid' % (tableNumber - 1, tableNumber)
                    clauselist += ' and '
                    tablelist += ','
                clauselist += 'w%d.wordid=%d' % (tableNumber, wordid)
                fieldlist += ',w%d.location' % tableNumber
                tablelist += 'wordlocation w%d' % tableNumber
                tableNumber += 1

        res = self.con.execute('select %s from %s where %s' % (fieldlist, tablelist, clauselist))
        rows = [row for row in res]
        return rows,wordids

    def getScoredList(self, rows, wordids):
        totalscores = dict([(row[0], 0) for row in rows])

        # To put the scoring functions
        weights = []

        for weight,scores in weights:
            for urlid in totalscores:
                totalscores[urlid] += scores[urlid] * weight

        return totalscores

    def getUrlName(self, urlid):
        res=self.con.execute('select url from urllist where rowid=%d' % urlid).fetchone()
        if res!=None:
            return res[0]
        return None

    def query(self, q):
        rows, wordids = self.getMatchRows(q)
        scores = self.getScoredList(rows, wordids)
        rankedScores = sorted([(score, urlid) for (urlid, score) in scores.items()], reverse=1)
        for score,urlid in rankedScores[0:10]:
            print '%f\t%s' % (score, self.getUrlName(urlid))

    def normalizeScores(self, scores, smallIsBetter=0):
        vsmall = 0.00001
        maxScore = max(scores.values())
        maxScore = max(maxScore, vsmall)
        if smallIsBetter:
            return dict([(u, 1 - float(c) / maxScore) for (u.c) in scores])
        else:
            return dict([(u, float(c) / maxScore) for (u.c) in scores])

    def frequencyScore(self, rows):

# test git with ssh
<commit_msg>searchengine frequency score<commit_after>import re
import urllib2
import sqlite3
from bs4 import BeautifulSoup
from urlparse import urljoin
from urlparse import urlparse

ignoreWords=set(['the','of','to','and','a','in','is','it'])

class crawler:
    # Initialize the crawler with the name of database
    def __init__(self,dbName):
        self.con=sqlite3.connect(dbName)

    def __del__(self):
        self.con.close()

    def dbCommit(self):
        self.con.commit()

    # Auxilliary function for getting an entry id and adding
    # it if it's not present
    def getEntryId(self,table,field,value,createNew=True):
        cur=self.con.execute("select rowid from %s where %s='%s'" % (table,field,value))
        res=cur.fetchone()
        if res==None:
            cur=self.con.execute("insert into %s(%s) values('%s')" % (table,field,value))
            return cur.lastrowid
        return res[0]

    # Index an individual page
    def addToIndex(self,url,soup):
        if self.isIndexed(url):
            return
        print 'Indexing %s' % url
        urlid=self.getEntryId('urllist','url',url)

        text=self.getTextOnly(soup)
        words=self.separateWords(text)
        for i in range(len(words)):
            word=words[i]
            if word in ignoreWords:
                continue
            wordid=self.getEntryId('wordlist','word',word)
            self.con.execute('insert into wordlocation(urlid,wordid,location) values(%d,%d,%d)' % (urlid,wordid,i))

    # Extract the text from an HTML page (no tags)
    def getTextOnly(self,soup):
        v=soup.string
        if v == None:
            result=''
            for content in soup.contents:
                result+=self.getTextOnly(content)+'\n'
            return result
        return v.strip()

    # Separate the words by any non-whitespace character
    def separateWords(self,text):
        splitter=re.compile('[^a-zA-Z0-9_+-]')
        return [t for t in splitter.split(text) if t!='']

    # Return true if this url is already indexed
    def isIndexed(self,url):
        res=self.con.execute("select rowid from urllist where url='%s'" % url).fetchone()
        if res!=None:
            urlid=res[0]
            res=self.con.execute("select rowid from wordlocation where urlid=%d" % urlid).fetchone()
            return res!=None
        return False

    # Add a link between two pages
    def addLinkRef(self,urlFrom,urlTo,linkText):
        pass

    # Starting with a list of pages, do a breadth
    # first search to the given depth, indexing pages
    # as we go
    def crawl(self,pages,depth=2):
        for i in range(depth):
            newPages=set()
            for page in pages:
                try:
                    c=urllib2.urlopen(page)
                except:
                    print "Could not open %s" % page

                soup=BeautifulSoup(c, 'html.parser')
                self.addToIndex(page,soup)
                parsedUri=urlparse(page)
                scheme='{uri.scheme}'.format(uri=parsedUri)
                domain='{uri.scheme}://{uri.netloc}/'.format(uri=parsedUri)
                links=soup('a')
                for link in links:
                    if 'href' in dict(link.attrs):
                        uri=urljoin(domain,link['href'])
                        if uri.find("'")!=-1: continue
                        uri=uri.split('#')[0]
                        if (scheme == 'http' or scheme == 'https') and not self.isIndexed(uri):
                            newPages.add(uri)
                        linkText=self.getTextOnly(link)
                        self.addLinkRef(page,uri,linkText)
                self.dbCommit()

            pages=newPages

    # Create the database tables
    def createIndexTables(self):
        self.con.execute('create table urllist(url)')
        self.con.execute('create table wordlist(word)')
        self.con.execute('create table wordlocation(urlid,wordid,location)')
        self.con.execute('create table link(fromid,toid)')
        self.con.execute('create table linkwords(wordid,linkid)')
        self.con.execute('create index urlidx on urllist(url)')
        self.con.execute('create index wordidx on wordlist(word)')
        self.con.execute('create index wordurlidx on wordlocation(wordid)')
        self.con.execute('create index urltoidx on link(toid)')
        self.con.execute('create index urlfromidx on link(fromid)')
        self.dbCommit()

class searcher:
    def __init__(self,dbName):
        self.con=sqlite3.connect(dbName)

    def __del__(self):
        self.con.close()

    def getMatchRows(self,q):
        fieldlist = 'w0.urlid'
        tablelist = ''
        clauselist = ''
        wordids = []

        words = q.split(' ')
        tableNumber = 0

        for word in words:
            res=self.con.execute("select rowid from wordlist where word='%s'" % word).fetchone()
            if res!=None:
                wordid = res[0]
                wordids.append(wordid)

                if tableNumber > 0:
                    clauselist += ' and w%d.urlid=w%d.urlid' % (tableNumber - 1, tableNumber)
                    clauselist += ' and '
                    tablelist += ','
                clauselist += 'w%d.wordid=%d' % (tableNumber, wordid)
                fieldlist += ',w%d.location' % tableNumber
                tablelist += 'wordlocation w%d' % tableNumber
                tableNumber += 1

        res = self.con.execute('select %s from %s where %s' % (fieldlist, tablelist, clauselist))
        rows = [row for row in res]
        return rows,wordids

    def getScoredList(self, rows, wordids):
        totalscores = dict([(row[0], 0) for row in rows])

        # To put the scoring functions
        weights = [(1.0, self.frequencyScore(rows))]

        for weight,scores in weights:
            for urlid in totalscores:
                totalscores[urlid] += scores[urlid] * weight

        return totalscores

    def getUrlName(self, urlid):
        res=self.con.execute('select url from urllist where rowid=%d' % urlid).fetchone()
        if res!=None:
            return res[0]
        return None

    def query(self, q):
        rows, wordids = self.getMatchRows(q)
        scores = self.getScoredList(rows, wordids)
        rankedScores = sorted([(score, urlid) for urlid,score in scores.items()], reverse=1)
        for score,urlid in rankedScores[0:10]:
            print '%f\t%s' % (score, self.getUrlName(urlid))

    def normalizeScores(self, scores, smallIsBetter=0):
        vsmall = 0.00001
        maxScore = max(scores.values())
        maxScore = max(maxScore, vsmall)
        if smallIsBetter:
            return dict([(u, 1 - float(c) / maxScore) for u,c in scores.items()])
        else:
            return dict([(u, float(c) / maxScore) for u,c in scores.items()])

    def frequencyScore(self, rows):
        counts = dict([(row[0], 0) for row in rows])
        for row in rows:
            counts[row[0]] += 1
        return self.normalizeScores(counts)

<|endoftext|>


<commit_before># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os
import time
from datetime import datetime
import random
import string
import webbrowser
import json
from parlai.core.agents import create_agent_from_shared
from .setup_aws import rds_db_name, rds_username, rds_password, setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials
from .data_model import Message, init_database, send_new_message, get_new_messages, get_pending_approval_count, get_all_approval_status

def _get_random_alphanumeric_string(N):
    return ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(N))

def setup_relay(task_config, num_hits, is_sandbox):
    """Sets up relay server and returns a database session object which can be used to poll
    new messages and send messages
    """
    # set up relay server
    rds_host, mturk_chat_url_template, mturk_approval_url_template = setup_aws(task_config, num_hits, is_sandbox)

    db_engine, db_session_maker = init_database(rds_host, rds_db_name, rds_username, rds_password)
    db_session = db_session_maker()

    return db_session, mturk_chat_url_template, mturk_approval_url_template

def create_hits(opt, task_config, task_module_name, bot, num_hits, hit_reward, is_sandbox=False, chat_page_only=False, verbose=False):
    print("\nYou are going to allow workers from Amazon Mechanical Turk to chat with your dialog model running on your local machine.\nDuring this process, Internet connection is required, and you cannot close your laptop or put your computer into sleep or standby mode.\n")
    key_input = input("Please press Enter to continue:")
    print("")

    setup_aws_credentials()
    if not check_mturk_balance(num_hits=num_hits, hit_reward=hit_reward, is_sandbox=is_sandbox):
        return

    task_group_id = str(int(time.time())) + '_' + _get_random_alphanumeric_string(10) # Random string to further avoid collision

    print('Setting up MTurk backend...')
    db_session, mturk_chat_url_template, mturk_approval_url_template = setup_relay(task_config, num_hits, is_sandbox)

    worker_agent_id = task_config['worker_agent_id']
    bot_agent_id = bot.getID()
    cids = range(1, num_hits+1)
    cid_map = {cid: i for i, cid in enumerate(cids)}
    c_done_map = {cid: False for cid in cids}
    logs = {cid: [] for cid in cids}

    shared = bot.share()
    bots = []
    last_message_id = -1

    # If the bot needs to send the first message of the conversation, it will send it here
    for cid in cids:
        new_bot = create_agent_from_shared(shared)
        new_bot.conversation_id = cid
        bots.append(new_bot)
        response = new_bot.act()
        if response:
            if response.get('episode_done', False):
                c_done_map[cid] = True
            if verbose:
                print('Conversation '+str(cid)+' - Bot says: ' + str(response))
            logs[cid].append(response)
            new_message_object = send_new_message(
                db_session=db_session,
                task_group_id=task_group_id,
                conversation_id=cid,
                agent_id=bot_agent_id,
                message_text=response.get('text', None),
                reward=response.get('reward', None),
                episode_done=response.get('episode_done', False),
            )
            if new_message_object.id > last_message_id:
                last_message_id = new_message_object.id

    hits_created = False
    conversations_remaining = set(cids)

    # Main loop for polling and handling new messages
    while len(conversations_remaining) > 0:
        conversation_dict, new_last_message_id = get_new_messages(
            db_session=db_session,
            task_group_id=task_group_id,
            after_message_id=last_message_id,
            excluded_agent_id=bot_agent_id,
        )

        if new_last_message_id:
            last_message_id = new_last_message_id

        time.sleep(1)

        for conversation_id, new_messages in conversation_dict.items():
            if conversation_id in conversations_remaining and len(new_messages) > 0:
                agent = bots[cid_map[conversation_id]]
                for new_message in new_messages:
                    if verbose:
                        print('Conversation '+str(conversation_id)+' - Bot received: ' + str(new_message))
                    logs[conversation_id].append(new_message)
                    agent.observe(new_message)
                    if new_message.get('episode_done', False) or c_done_map[conversation_id]:
                        # We're done here
                        conversations_remaining.remove(conversation_id)
                        print('Conversation '+str(conversation_id)+' is DONE!\n')
                    else:
                        # Agent still needs to reply
                        response = agent.act()
                        if response:
                            if response.get('episode_done', False):
                                c_done_map[conversation_id] = True
                            if verbose:
                                print('Conversation '+str(conversation_id)+' - Bot says: ' + str(response))
                            logs[conversation_id].append(response)
                            send_new_message(
                                db_session=db_session,
                                task_group_id=task_group_id,
                                conversation_id=conversation_id,
                                agent_id=bot_agent_id,
                                message_text=response.get('text', None),
                                reward=response.get('reward', None),
                                episode_done=response.get('episode_done', False),
                            )

        # We don't create new HITs until this point, so that the HIT page will always have the conversation fully populated.
        if not hits_created:
            print('Creating HITs...')
            hit_type_id = create_hit_type(
                hit_title=task_config['hit_title'],
                hit_description=task_config['hit_description'] + ' (ID: ' + task_group_id + ')',
                hit_keywords=task_config['hit_keywords'],
                hit_reward=hit_reward,
                is_sandbox=is_sandbox
            )
            mturk_chat_url = None
            mturk_page_url = None
            for cid in cids:
                mturk_chat_url = mturk_chat_url_template.replace('{{task_group_id}}', str(task_group_id)).replace('{{conversation_id}}', str(cid)).replace('{{cur_agent_id}}', str(worker_agent_id))
                if not chat_page_only:
                    mturk_page_url = create_hit_with_hit_type(
                        page_url=mturk_chat_url,
                        hit_type_id=hit_type_id,
                        is_sandbox=is_sandbox
                    )

            print("MTurk setup done.\n")
            if chat_page_only:
                webbrowser.open(mturk_chat_url)
            else:
                print("Link to your HIT: " + mturk_page_url + "\n")
                print("Waiting for Turkers to complete the tasks... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
            hits_created = True

    while get_pending_approval_count(db_session, task_group_id) != num_hits:
        time.sleep(2)

    mturk_approval_url = mturk_approval_url_template.replace('{{task_group_id}}', str(task_group_id)).replace('{{cur_agent_id}}', str(worker_agent_id))
    print("\nAll HITs are done! Please go to the following link to approve/reject them (they will be auto-approved in 4 weeks if no action is taken):\n")
    print(mturk_approval_url)
    print("")

    approval_status_dict = {cid: '' for cid in cids}
    # Loop for checking approval status
    while get_pending_approval_count(db_session, task_group_id) > 0:
        time.sleep(2)

    print("Approvals are done!")

    for hit_info in get_all_approval_status(db_session, task_group_id):
        conversation_id = hit_info.conversation_id
        approval_status_dict[conversation_id] = hit_info.approval_status

    logs_approved = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'approved'}
    logs_rejected = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'rejected'}

    # Saving logs to file
    # Log format: {conversation_id: [list of messages in the conversation]}
    mturk_log_path = opt['mturk_log_path']
    task_group_path = mturk_log_path + task_module_name + '_' + datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + '/'
    os.makedirs(task_group_path)
    with open(task_group_path+'approved.json', 'w') as file:
        file.write(json.dumps(logs_approved))
    with open(task_group_path+'rejected.json', 'w') as file:
        file.write(json.dumps(logs_rejected))

    print("All conversations are saved to "+opt['mturk_log_path']+" in JSON format.\n")
<commit_msg>improved mturk instruction<commit_after># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os
import time
from datetime import datetime
import random
import string
import webbrowser
import json
from parlai.core.agents import create_agent_from_shared
from .setup_aws import rds_db_name, rds_username, rds_password, setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials
from .data_model import Message, init_database, send_new_message, get_new_messages, get_pending_approval_count, get_all_approval_status

def _get_random_alphanumeric_string(N):
    return ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(N))

def setup_relay(task_config, num_hits, is_sandbox):
    """Sets up relay server and returns a database session object which can be used to poll
    new messages and send messages
    """
    # set up relay server
    rds_host, mturk_chat_url_template, mturk_approval_url_template = setup_aws(task_config, num_hits, is_sandbox)

    db_engine, db_session_maker = init_database(rds_host, rds_db_name, rds_username, rds_password)
    db_session = db_session_maker()

    return db_session, mturk_chat_url_template, mturk_approval_url_template

def create_hits(opt, task_config, task_module_name, bot, num_hits, hit_reward, is_sandbox=False, chat_page_only=False, verbose=False):
    print("\nYou are going to allow workers from Amazon Mechanical Turk to chat with your dialog model running on your local machine.\nDuring this process, Internet connection is required, and you should turn off your computer's auto-sleep feature.\n")
    key_input = input("Please press Enter to continue:")
    print("")

    setup_aws_credentials()
    if not check_mturk_balance(num_hits=num_hits, hit_reward=hit_reward, is_sandbox=is_sandbox):
        return

    task_group_id = str(int(time.time())) + '_' + _get_random_alphanumeric_string(10) # Random string to further avoid collision

    print('Setting up MTurk backend...')
    db_session, mturk_chat_url_template, mturk_approval_url_template = setup_relay(task_config, num_hits, is_sandbox)

    worker_agent_id = task_config['worker_agent_id']
    bot_agent_id = bot.getID()
    cids = range(1, num_hits+1)
    cid_map = {cid: i for i, cid in enumerate(cids)}
    c_done_map = {cid: False for cid in cids}
    logs = {cid: [] for cid in cids}

    shared = bot.share()
    bots = []
    last_message_id = -1

    # If the bot needs to send the first message of the conversation, it will send it here
    for cid in cids:
        new_bot = create_agent_from_shared(shared)
        new_bot.conversation_id = cid
        bots.append(new_bot)
        response = new_bot.act()
        if response:
            if response.get('episode_done', False):
                c_done_map[cid] = True
            if verbose:
                print('Conversation '+str(cid)+' - Bot says: ' + str(response))
            logs[cid].append(response)
            new_message_object = send_new_message(
                db_session=db_session,
                task_group_id=task_group_id,
                conversation_id=cid,
                agent_id=bot_agent_id,
                message_text=response.get('text', None),
                reward=response.get('reward', None),
                episode_done=response.get('episode_done', False),
            )
            if new_message_object.id > last_message_id:
                last_message_id = new_message_object.id

    hits_created = False
    conversations_remaining = set(cids)

    # Main loop for polling and handling new messages
    while len(conversations_remaining) > 0:
        conversation_dict, new_last_message_id = get_new_messages(
            db_session=db_session,
            task_group_id=task_group_id,
            after_message_id=last_message_id,
            excluded_agent_id=bot_agent_id,
        )

        if new_last_message_id:
            last_message_id = new_last_message_id

        time.sleep(1)

        for conversation_id, new_messages in conversation_dict.items():
            if conversation_id in conversations_remaining and len(new_messages) > 0:
                agent = bots[cid_map[conversation_id]]
                for new_message in new_messages:
                    if verbose:
                        print('Conversation '+str(conversation_id)+' - Bot received: ' + str(new_message))
                    logs[conversation_id].append(new_message)
                    agent.observe(new_message)
                    if new_message.get('episode_done', False) or c_done_map[conversation_id]:
                        # We're done here
                        conversations_remaining.remove(conversation_id)
                        print('Conversation '+str(conversation_id)+' is DONE!\n')
                    else:
                        # Agent still needs to reply
                        response = agent.act()
                        if response:
                            if response.get('episode_done', False):
                                c_done_map[conversation_id] = True
                            if verbose:
                                print('Conversation '+str(conversation_id)+' - Bot says: ' + str(response))
                            logs[conversation_id].append(response)
                            send_new_message(
                                db_session=db_session,
                                task_group_id=task_group_id,
                                conversation_id=conversation_id,
                                agent_id=bot_agent_id,
                                message_text=response.get('text', None),
                                reward=response.get('reward', None),
                                episode_done=response.get('episode_done', False),
                            )

        # We don't create new HITs until this point, so that the HIT page will always have the conversation fully populated.
        if not hits_created:
            print('Creating HITs...')
            hit_type_id = create_hit_type(
                hit_title=task_config['hit_title'],
                hit_description=task_config['hit_description'] + ' (ID: ' + task_group_id + ')',
                hit_keywords=task_config['hit_keywords'],
                hit_reward=hit_reward,
                is_sandbox=is_sandbox
            )
            mturk_chat_url = None
            mturk_page_url = None
            for cid in cids:
                mturk_chat_url = mturk_chat_url_template.replace('{{task_group_id}}', str(task_group_id)).replace('{{conversation_id}}', str(cid)).replace('{{cur_agent_id}}', str(worker_agent_id))
                if not chat_page_only:
                    mturk_page_url = create_hit_with_hit_type(
                        page_url=mturk_chat_url,
                        hit_type_id=hit_type_id,
                        is_sandbox=is_sandbox
                    )

            print("MTurk setup done.\n")
            if chat_page_only:
                webbrowser.open(mturk_chat_url)
            else:
                print("Link to your HIT: " + mturk_page_url + "\n")
                print("Waiting for Turkers to complete the tasks... (Please don't close your laptop or put your computer into sleep or standby mode.)\n")
            hits_created = True

    while get_pending_approval_count(db_session, task_group_id) != num_hits:
        time.sleep(2)

    mturk_approval_url = mturk_approval_url_template.replace('{{task_group_id}}', str(task_group_id)).replace('{{cur_agent_id}}', str(worker_agent_id))
    print("\nAll HITs are done! Please go to the following link to approve/reject them (they will be auto-approved in 4 weeks if no action is taken):\n")
    print(mturk_approval_url)
    print("")

    approval_status_dict = {cid: '' for cid in cids}
    # Loop for checking approval status
    while get_pending_approval_count(db_session, task_group_id) > 0:
        time.sleep(2)

    print("Approvals are done!")

    for hit_info in get_all_approval_status(db_session, task_group_id):
        conversation_id = hit_info.conversation_id
        approval_status_dict[conversation_id] = hit_info.approval_status

    logs_approved = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'approved'}
    logs_rejected = {cid:log for (cid,log) in logs.items() if approval_status_dict[cid] == 'rejected'}

    # Saving logs to file
    # Log format: {conversation_id: [list of messages in the conversation]}
    mturk_log_path = opt['mturk_log_path']
    task_group_path = mturk_log_path + task_module_name + '_' + datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + '/'
    os.makedirs(task_group_path)
    with open(task_group_path+'approved.json', 'w') as file:
        file.write(json.dumps(logs_approved))
    with open(task_group_path+'rejected.json', 'w') as file:
        file.write(json.dumps(logs_rejected))

    print("All conversations are saved to "+opt['mturk_log_path']+" in JSON format.\n")
<|endoftext|>