<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import re

import rml_parse
from report import report_sxw
from common_report_header import common_report_header

class third_party_ledger(rml_parse.rml_parse, common_report_header):

    def __init__(self, cr, uid, name, context=None):
        super(third_party_ledger, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'time': time,
            'lines': self.lines,
            'sum_debit_partner': self._sum_debit_partner,
            'sum_credit_partner': self._sum_credit_partner,
            'sum_debit': self._sum_debit,
            'sum_credit': self._sum_credit,
            'get_currency': self._get_currency,
            'comma_me' : self.comma_me,
            'get_start_period': self.get_start_period,
            'get_end_period': self.get_end_period,
            'get_account': self._get_account,
            'get_filter': self._get_filter,
            'get_start_date': self._get_start_date,
            'get_end_date': self._get_end_date,
            'get_fiscalyear': self._get_fiscalyear,
            'get_start_date':self._get_start_date,
            'get_end_date': self._get_end_date,
            'get_journal': self._get_journal,
            'get_partners':self._get_partners,
            'get_intial_balance':self._get_intial_balance,
            'display_initial_balance':self._display_initial_balance,
            'display_currency':self._display_currency,
        })

    def set_context(self, objects, data, ids, report_type=None):
        self.query = data['form'].get('query_line', '')
        self.init_query = data['form'].get('initial_bal_query', '')
        self.reconcil = data['form'].get('reconcil', True)
        self.initial_balance = data['form'].get('initial_balance', True)
        self.result_selection = data['form'].get('result_selection', 'customer')
        self.amount_currency = data['form'].get('amount_currency', False)
        PARTNER_REQUEST = ''
        if (data['model'] == 'res.partner'):
            ## Si on imprime depuis les partenaires
            if ids:
                PARTNER_REQUEST =  "AND line.partner_id IN %s",(tuple(ids),)
        if self.result_selection == 'supplier':
            self.ACCOUNT_TYPE = ['payable']
        elif self.result_selection == 'customer':
            self.ACCOUNT_TYPE = ['receivable']
        else:
            self.ACCOUNT_TYPE = ['payable','receivable']

        self.cr.execute(
            "SELECT a.id " \
            "FROM account_account a " \
            "LEFT JOIN account_account_type t " \
                "ON (a.type=t.code) " \
                'WHERE a.type IN %s' \
                "AND a.active", (tuple(self.ACCOUNT_TYPE), ))
        self.account_ids = [a for (a,) in self.cr.fetchall()]
        partner_to_use = []
        self.cr.execute(
                "SELECT DISTINCT l.partner_id " \
                "FROM account_move_line AS l, account_account AS account " \
                "WHERE l.partner_id IS NOT NULL " \
                    "AND l.account_id = account.id " \
                    "AND " + self.query +" " \
                    "AND l.account_id IN %s " \
                    " " + PARTNER_REQUEST + " " \
                    "AND account.active " ,
                (tuple(self.account_ids),))

        res = self.cr.dictfetchall()
        for res_line in res:
            partner_to_use.append(res_line['partner_id'])
        new_ids = partner_to_use
        self.partner_ids = new_ids
        objects = self.pool.get('res.partner').browse(self.cr, self.uid, new_ids)
        return super(third_party_ledger, self).set_context(objects, data, new_ids, report_type)

    def comma_me(self, amount):
        if type(amount) is float :
            amount = str('%.2f'%amount)
        else :
            amount = str(amount)
        if (amount == '0'):
             return ' '
        orig = amount
        new = re.sub("^(-?\d+)(\d{3})", "\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)

    def lines(self, partner):
        full_account = []
        if self.reconcil:
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND l.reconcile_id IS NULL"
        self.cr.execute(
            "SELECT l.id,l.date,j.code, l.ref, m.name as move_name, l.name, l.debit, l.credit,l.amount_currency,c.code AS currency_code " \
            "FROM account_move_line l " \
            "LEFT JOIN account_journal j " \
                "ON (l.journal_id = j.id) " \
            "LEFT JOIN res_currency c on (l.currency_id=c.id)" \
            "LEFT JOIN account_move m on (m.id=l.move_id)" \
            "WHERE l.partner_id = %s " \
                "AND l.account_id IN %s AND " + self.query +" " \
                " " + RECONCILE_TAG + " "\
                "ORDER BY l.id",
                (partner.id, tuple(self.account_ids)))
        res = self.cr.dictfetchall()
        sum = 0.0
        for r in res:
            sum = r['debit'] - r['credit']
            r['progress'] = sum
            full_account.append(r)
        return full_account

    def _get_intial_balance(self, partner):
        self.cr.execute(
            "SELECT COALESCE(SUM(l.debit),0.0), COALESCE(SUM(l.credit),0.0), COALESCE(sum(debit-credit), 0.0) " \
            "FROM account_move_line AS l " \
            "WHERE partner_id = %s " \
            "AND account_id IN %s" \
            "AND reconcile_id IS NULL  " \
            "AND " + self.init_query + "  ",
            (partner.id, tuple(self.account_ids)))
        return self.cr.fetchall()

    def _sum_debit_partner(self, partner):
        result_tmp = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        self.cr.execute(
                "SELECT sum(debit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id = %s " \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                (partner.id,tuple(self.account_ids),))

        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp

    def _sum_credit_partner(self, partner):
        result_tmp = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        self.cr.execute(
                "SELECT sum(credit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id=%s " \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                (partner.id, tuple(self.account_ids),))

        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp

    def _sum_debit(self):
        if not self.ids:
            return 0.0
        result_tmp = 0.0
        result_init = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        if self.initial_balance:
            self.cr.execute(
                    "SELECT sum(debit) " \
                    "FROM account_move_line AS l " \
                    "WHERE partner_id IN %s" \
                        "AND account_id IN %s" \
                        "AND reconcile_id IS NULL " \
                        "AND " + self.init_query + " ",
                    (tuple(self.partner_ids), tuple(self.account_ids)))
            contemp = self.cr.fetchone()
            if contemp != None:
                result_init = contemp[0] or 0.0
            else:
                result_init = result_tmp + 0.0

        self.cr.execute(
                "SELECT sum(debit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id IN %s" \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                    (tuple(self.partner_ids), tuple(self.account_ids),))
        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp  + result_init

    def _sum_credit(self):
        if not self.ids:
            return 0.0
        result_tmp = 0.0
        result_init = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        if self.initial_balance:
            self.cr.execute(
                    "SELECT sum(credit) " \
                    "FROM account_move_line AS l " \
                    "WHERE partner_id IN %s" \
                        "AND account_id IN %s" \
                        "AND reconcile_id IS NULL " \
                        "AND " + self.init_query + " ",
                    (tuple(self.partner_ids), tuple(self.account_ids)))
            contemp = self.cr.fetchone()
            if contemp != None:
                result_init = contemp[0] or 0.0
            else:
                result_init = result_tmp + 0.0

        self.cr.execute(
                "SELECT sum(credit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id  IN %s" \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                    (tuple(self.partner_ids), tuple(self.account_ids),))
        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0

        return result_tmp  + result_init
#
    def _get_partners(self):
        if self.result_selection == 'customer':
            return 'Receivable Accounts'
        elif self.result_selection == 'supplier':
            return 'Payable Accounts'
        elif self.result_selection == 'customer_supplier':
            return 'Receivable and Payable Accounts'
        return ''

    def _sum_currency_amount_account(self, account, form):
        self._set_get_account_currency_code(account.id)
        self.cr.execute("SELECT sum(aml.amount_currency) FROM account_move_line as aml,res_currency as rc WHERE aml.currency_id = rc.id AND aml.account_id= %s ", (account.id,))
        total = self.cr.fetchone()
        if self.account_currency:
            return_field = str(total[0]) + self.account_currency
            return return_field
        else:
            currency_total = self.tot_currency = 0.0
            return currency_total

    def _display_initial_balance(self, data):
         if self.initial_balance:
             return True
         return False

    def _display_currency(self, data):
         if self.amount_currency :
             return True
         return False

report_sxw.report_sxw('report.account.third_party_ledger', 'res.partner',
        'addons/account/report/account_partner_ledger.rml',parser=third_party_ledger,
        header='internal')

report_sxw.report_sxw('report.account.third_party_ledger_other', 'res.partner',
        'addons/account/report/account_partner_ledger_other.rml',parser=third_party_ledger,
        header='internal')

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] Account: Partner ledger report: Add sort by date for lines<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import re

import rml_parse
from report import report_sxw
from common_report_header import common_report_header

class third_party_ledger(rml_parse.rml_parse, common_report_header):

    def __init__(self, cr, uid, name, context=None):
        super(third_party_ledger, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'time': time,
            'lines': self.lines,
            'sum_debit_partner': self._sum_debit_partner,
            'sum_credit_partner': self._sum_credit_partner,
            'sum_debit': self._sum_debit,
            'sum_credit': self._sum_credit,
            'get_currency': self._get_currency,
            'comma_me' : self.comma_me,
            'get_start_period': self.get_start_period,
            'get_end_period': self.get_end_period,
            'get_account': self._get_account,
            'get_filter': self._get_filter,
            'get_start_date': self._get_start_date,
            'get_end_date': self._get_end_date,
            'get_fiscalyear': self._get_fiscalyear,
            'get_start_date':self._get_start_date,
            'get_end_date': self._get_end_date,
            'get_journal': self._get_journal,
            'get_partners':self._get_partners,
            'get_intial_balance':self._get_intial_balance,
            'display_initial_balance':self._display_initial_balance,
            'display_currency':self._display_currency,
        })

    def set_context(self, objects, data, ids, report_type=None):
        self.query = data['form'].get('query_line', '')
        self.init_query = data['form'].get('initial_bal_query', '')
        self.reconcil = data['form'].get('reconcil', True)
        self.initial_balance = data['form'].get('initial_balance', True)
        self.result_selection = data['form'].get('result_selection', 'customer')
        self.amount_currency = data['form'].get('amount_currency', False)
        PARTNER_REQUEST = ''
        if (data['model'] == 'res.partner'):
            ## Si on imprime depuis les partenaires
            if ids:
                PARTNER_REQUEST =  "AND line.partner_id IN %s",(tuple(ids),)
        if self.result_selection == 'supplier':
            self.ACCOUNT_TYPE = ['payable']
        elif self.result_selection == 'customer':
            self.ACCOUNT_TYPE = ['receivable']
        else:
            self.ACCOUNT_TYPE = ['payable','receivable']

        self.cr.execute(
            "SELECT a.id " \
            "FROM account_account a " \
            "LEFT JOIN account_account_type t " \
                "ON (a.type=t.code) " \
                'WHERE a.type IN %s' \
                "AND a.active", (tuple(self.ACCOUNT_TYPE), ))
        self.account_ids = [a for (a,) in self.cr.fetchall()]
        partner_to_use = []
        self.cr.execute(
                "SELECT DISTINCT l.partner_id " \
                "FROM account_move_line AS l, account_account AS account " \
                "WHERE l.partner_id IS NOT NULL " \
                    "AND l.account_id = account.id " \
                    "AND " + self.query +" " \
                    "AND l.account_id IN %s " \
                    " " + PARTNER_REQUEST + " " \
                    "AND account.active " ,
                (tuple(self.account_ids),))

        res = self.cr.dictfetchall()
        for res_line in res:
            partner_to_use.append(res_line['partner_id'])
        new_ids = partner_to_use
        self.partner_ids = new_ids
        objects = self.pool.get('res.partner').browse(self.cr, self.uid, new_ids)
        return super(third_party_ledger, self).set_context(objects, data, new_ids, report_type)

    def comma_me(self, amount):
        if type(amount) is float :
            amount = str('%.2f'%amount)
        else :
            amount = str(amount)
        if (amount == '0'):
             return ' '
        orig = amount
        new = re.sub("^(-?\d+)(\d{3})", "\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)

    def lines(self, partner):
        full_account = []
        if self.reconcil:
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND l.reconcile_id IS NULL"
        self.cr.execute(
            "SELECT l.id,l.date,j.code, l.ref, m.name as move_name, l.name, l.debit, l.credit,l.amount_currency,c.code AS currency_code " \
            "FROM account_move_line l " \
            "LEFT JOIN account_journal j " \
                "ON (l.journal_id = j.id) " \
            "LEFT JOIN res_currency c on (l.currency_id=c.id)" \
            "LEFT JOIN account_move m on (m.id=l.move_id)" \
            "WHERE l.partner_id = %s " \
                "AND l.account_id IN %s AND " + self.query +" " \
                " " + RECONCILE_TAG + " "\
                "ORDER BY l.date",
                (partner.id, tuple(self.account_ids)))
        res = self.cr.dictfetchall()
        sum = 0.0
        for r in res:
            sum = r['debit'] - r['credit']
            r['progress'] = sum
            full_account.append(r)
        return full_account

    def _get_intial_balance(self, partner):
        self.cr.execute(
            "SELECT COALESCE(SUM(l.debit),0.0), COALESCE(SUM(l.credit),0.0), COALESCE(sum(debit-credit), 0.0) " \
            "FROM account_move_line AS l " \
            "WHERE partner_id = %s " \
            "AND account_id IN %s" \
            "AND reconcile_id IS NULL  " \
            "AND " + self.init_query + "  ",
            (partner.id, tuple(self.account_ids)))
        return self.cr.fetchall()

    def _sum_debit_partner(self, partner):
        result_tmp = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        self.cr.execute(
                "SELECT sum(debit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id = %s " \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                (partner.id,tuple(self.account_ids),))

        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp

    def _sum_credit_partner(self, partner):
        result_tmp = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        self.cr.execute(
                "SELECT sum(credit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id=%s " \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                (partner.id, tuple(self.account_ids),))

        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp

    def _sum_debit(self):
        if not self.ids:
            return 0.0
        result_tmp = 0.0
        result_init = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        if self.initial_balance:
            self.cr.execute(
                    "SELECT sum(debit) " \
                    "FROM account_move_line AS l " \
                    "WHERE partner_id IN %s" \
                        "AND account_id IN %s" \
                        "AND reconcile_id IS NULL " \
                        "AND " + self.init_query + " ",
                    (tuple(self.partner_ids), tuple(self.account_ids)))
            contemp = self.cr.fetchone()
            if contemp != None:
                result_init = contemp[0] or 0.0
            else:
                result_init = result_tmp + 0.0

        self.cr.execute(
                "SELECT sum(debit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id IN %s" \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                    (tuple(self.partner_ids), tuple(self.account_ids),))
        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0
        return result_tmp  + result_init

    def _sum_credit(self):
        if not self.ids:
            return 0.0
        result_tmp = 0.0
        result_init = 0.0
        if self.reconcil :
            RECONCILE_TAG = " "
        else:
            RECONCILE_TAG = "AND reconcile_id IS NULL"
        if self.initial_balance:
            self.cr.execute(
                    "SELECT sum(credit) " \
                    "FROM account_move_line AS l " \
                    "WHERE partner_id IN %s" \
                        "AND account_id IN %s" \
                        "AND reconcile_id IS NULL " \
                        "AND " + self.init_query + " ",
                    (tuple(self.partner_ids), tuple(self.account_ids)))
            contemp = self.cr.fetchone()
            if contemp != None:
                result_init = contemp[0] or 0.0
            else:
                result_init = result_tmp + 0.0

        self.cr.execute(
                "SELECT sum(credit) " \
                "FROM account_move_line AS l " \
                "WHERE partner_id  IN %s" \
                    "AND account_id IN %s" \
                    " " + RECONCILE_TAG + " " \
                    "AND " + self.query + " " ,
                    (tuple(self.partner_ids), tuple(self.account_ids),))
        contemp = self.cr.fetchone()
        if contemp != None:
            result_tmp = contemp[0] or 0.0
        else:
            result_tmp = result_tmp + 0.0

        return result_tmp  + result_init
#
    def _get_partners(self):
        if self.result_selection == 'customer':
            return 'Receivable Accounts'
        elif self.result_selection == 'supplier':
            return 'Payable Accounts'
        elif self.result_selection == 'customer_supplier':
            return 'Receivable and Payable Accounts'
        return ''

    def _sum_currency_amount_account(self, account, form):
        self._set_get_account_currency_code(account.id)
        self.cr.execute("SELECT sum(aml.amount_currency) FROM account_move_line as aml,res_currency as rc WHERE aml.currency_id = rc.id AND aml.account_id= %s ", (account.id,))
        total = self.cr.fetchone()
        if self.account_currency:
            return_field = str(total[0]) + self.account_currency
            return return_field
        else:
            currency_total = self.tot_currency = 0.0
            return currency_total

    def _display_initial_balance(self, data):
         if self.initial_balance:
             return True
         return False

    def _display_currency(self, data):
         if self.amount_currency :
             return True
         return False

report_sxw.report_sxw('report.account.third_party_ledger', 'res.partner',
        'addons/account/report/account_partner_ledger.rml',parser=third_party_ledger,
        header='internal')

report_sxw.report_sxw('report.account.third_party_ledger_other', 'res.partner',
        'addons/account/report/account_partner_ledger_other.rml',parser=third_party_ledger,
        header='internal')

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

from abc import abstractmethod, ABCMeta
from contextlib import contextmanager

try:
    from urllib.request import urlopen, Request, quote
    from urllib.parse import urlencode
except ImportError:
    from urllib2 import urlopen, Request, quote
    from urllib import urlencode

__all__ = [
    'AbstractBaseAPIClient',
    'GoogleBookAPIClient',
]

class AbstractBaseAPIClient:
    """
    AbstractBaseAPIClient that specifies the abstractmethods
    necessary to propertly be handled.
    The AbstractBaseClass defines a minimal set of methods that establish
    the characteristic behavior for the APIClient.

    Code that discriminates based on Abstract methods can trust that
    those methods will always be present.
    """
    __metaclass__ = ABCMeta
    @abstractmethod
    def connect():
        pass
    @abstractmethod
    def reader():
        pass

class GoogleBookAPIClient(AbstractBaseAPIClient):
    """
    Implements the AbstractBaseAPIClient and talks to the google books
    api for querying and finding books.

    :return self: **GoogleBookAPIClient<self>**
    """

    def __init__(self, title, author=None, max_results=10, lang='en'):
        """
        :param title: str

        :param author: str
        :param max_results: int
        :param lang: str
        """
        self.title = title
        self.author = author
        self.max_results = max_results
        self.lang = lang

    @property
    def url(self):
        base = r'https://www.googleapis.com/books/v1/volumes'
        query = r'"{title}"'.format(title=self.title)
        if self.author:
            query = ' '.join([query, ':'.join(['inauthor', self.author])])
        params = urlencode({
            'q': query,
            'maxResults': self.max_results,
            'langRestrict': self.lang,
            })
        return '?'.join([base, params])

    @contextmanager
    def connect(self):
        """
        Context manager for HTTP Connection state and ensures proper handling
        of network sockets, sends a GET request.

        Exception is raised at the yield statement.

        :yield request: FileIO<Socket>
        """
        try:
            headers = {'User-Agent': 'Python'}
            request = urlopen(Request(self.url, headers=headers))
            yield request
        finally:
            request.close()

    def reader(self):
        """
        Reads raw text from the connection stream.
        Ensures proper exception handling.

        :return bytes: request
        """
        with self.connect() as request:
            if request.msg != 'OK':
                raise IOError
            request_stream = request.read().decode('utf-8')
        return request_stream

    def json(self):
        """
        :return dict: json
        """
        raw_text = self.reader()
        return json.loads(raw_text)
<commit_msg>metaclass documentation updated<commit_after># -*- coding: utf-8 -*-
import json

from abc import abstractmethod, ABCMeta, abstractproperty
from contextlib import contextmanager

try:
    from urllib.request import urlopen, Request
    from urllib.parse import urlencode
except ImportError:
    from urllib2 import urlopen, Request
    from urllib import urlencode

__all__ = [
    'AbstractBaseAPIClient',
    'GoogleBookAPIClient',
]

class AbstractBaseAPIClient:
    """
    AbstractBaseAPIClient that specifies the abstractmethods
    necessary to propertly be handled.
    The AbstractBaseClass defines a minimal set of methods that establish
    the characteristic behavior for the APIClient.

    Code that discriminates based on Abstract methods can trust that
    those methods will always be present.
    """
    __metaclass__ = ABCMeta

    @abstractproperty
    def url(self):
        """
        Routes to proper destination URL and proper encoding schemes.

        :return url: str
        """
        return

    @abstractmethod
    def connect(self):
        """
        Makes connection to the backend API and handles any exceptions.
        """
        return

    @abstractmethod
    def reader(self):
        """
        Reads content.

        :return text: str
        """
        return

class GoogleBookAPIClient(AbstractBaseAPIClient):
    """
    Implements the AbstractBaseAPIClient and talks to the google books
    api for querying and finding books.

    :return self: **GoogleBookAPIClient<self>**
    """

    def __init__(self, title, author=None, max_results=10, lang='en'):
        """
        :param title: str

        :param author: str
        :param max_results: int
        :param lang: str
        """
        self.title = title
        self.author = author
        self.max_results = max_results
        self.lang = lang

    @property
    def url(self):
        base = r'https://www.googleapis.com/books/v1/volumes'
        query = r'"{title}"'.format(title=self.title)
        if self.author:
            query = ' '.join([query, ':'.join(['inauthor', self.author])])
        params = urlencode({
            'q': query,
            'maxResults': self.max_results,
            'langRestrict': self.lang,
            })
        return '?'.join([base, params])

    @contextmanager
    def connect(self):
        """
        Context manager for HTTP Connection state and ensures proper handling
        of network sockets, sends a GET request.

        Exception is raised at the yield statement.

        :yield request: FileIO<Socket>
        """
        try:
            headers = {'User-Agent': 'Python'}
            request = urlopen(Request(self.url, headers=headers))
            yield request
        finally:
            request.close()

    def reader(self):
        """
        Reads raw text from the connection stream.
        Ensures proper exception handling.

        :return bytes: request
        """
        with self.connect() as request:
            if request.msg != 'OK':
                raise IOError
            request_stream = request.read().decode('utf-8')
        return request_stream

    def json(self):
        """
        :return dict: json
        """
        raw_text = self.reader()
        return json.loads(raw_text)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import scrapy
from whitebearautos.items import WhitebearautosItem
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.contrib.loader import ItemLoader
from scrapy.contrib.loader.processor import Compose, MapCompose
from w3lib.html import replace_escape_chars, remove_tags
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.http import Request
from scrapy.selector import Selector
import urlparse

class WhitebearautosspiderSpider(CrawlSpider):
    name = "whitebearautoSpider"
    allowed_domains = ["whitebearautos.com"]
    seed = 'http://www.whitebearautos.com/used-inventory/index.htm?start=%d'
    start_urls = [seed % i for i in [0, 35, 70, 105, 140, 175, 210, 245, 280, 315]]

    def parse(self, response):
        hxs = Selector(response)
        item_selector = hxs.xpath('//h2/a/@href').extract()
        models = hxs.xpath('//h2/a/span/text()').extract()
        miles = hxs.xpath('//*[@class="mileageValue"]/text()').extract()
        engines = hxs.xpath('//*[@class="engineValue"]/text()').extract()
        ext_colors = hxs.xpath('//*[@class="extColorValue"]/text()').extract()
        stock_numbers = hxs.xpath('//*[@class="stockNumberValue"]/text()').extract()

        for url, model, mile, engine, ext_color, stock_number in zip(item_selector, models, miles, engines, ext_colors, stock_numbers):
            yield Request(urlparse.urljoin(response.url, url),
                          callback=self.parse_item,
                          meta={'model': model, 'mile': mile, 'engine': engine, 'ext_color': ext_color, 'stock_number': stock_number},
                          )

    def parse_item(self, response):
        l = ItemLoader(item=WhitebearautosItem(), response=response)
        l.default_output_processor = MapCompose(lambda v: v.strip(), replace_escape_chars)

        year_selector = response.xpath('//h1/span/text()')[1].extract()[0:4]
        l.add_xpath('year', year_selector)
        l.add_value('model', response.meta['model'])
        l.add_xpath('price', '//*[@class="internet"][2]/span/text()[2]')
        l.add_value('miles', response.meta['mile'])
        l.add_value('engine', response.meta['engine'])
        vin_selector = response.xpath('//span[text()="VIN"]/following::dd[1]//text()').extract()
        l.add_value('vin', vin_selector)
        l.add_value('ext_color', response.meta['ext_color'])
        int_color_selector = response.xpath('//span[text()="Int. Color"]/following::dd[1]//text()').extract()
        l.add_value('int_color', int_color_selector)
        l.add_value('stock_number', response.meta['stock_number'])
        transmission_selector = response.xpath('//span[text()="Transmission"]/following::dd[1]//text()').extract()
        l.add_value('transmission', transmission_selector)
        l.add_xpath('marketing_text', '//*[@class="comments truncateComments"]/text()')
        l.add_value('url_of_the_vehicle_detail_page', response.url)
        l.add_xpath('vehicle_image', '/html/body/div[2]/div[1]/div/div[3]/div[3]/div/div[1]/div/div/div[1]/div[1]/img/@src')
        return l.load_item()
<commit_msg>deleted old spider file.<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import urlparse

from scrapy import Spider
from scrapy.http import Request
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose
from w3lib.html import replace_escape_chars, remove_tags

from whitebearautos.items import WhitebearautosItem

class WhitebearautosspiderSpider(Spider):
    name = 'whitebearautos'
    allowed_domains = ['whitebearautos.com']
    start_urls = ['http://www.whitebearautos.com/inventory/view/Used/']

    def parse(self, response):
        # process each car link
        urls = response.xpath('//h2/a/@href').extract()
        for url in urls:
            absolute_url = response.urljoin(url)
            request = Request(absolute_url, callback=self.parse_car)
            yield request

        # process next page
        next_page_url = response.xpath('.//*[@class="next"]/a/@href').extract_first()
        absolute_next_page_url = response.urljoin(next_page_url)
        request = Request(absolute_next_page_url)
        yield request

    def parse_car(self, response):
        l = ItemLoader(item=WhitebearautosItem(), response=response)
        l.default_output_processor = MapCompose(lambda v: v.strip(), replace_escape_chars)

        car_data = response.xpath('.//*[@class="cardata"]//text()').extract()
        if len(car_data) is 8:
            year, __, make, __, model, __, trim, __ = car_data
        elif len(car_data) is 7:
            year, __, make, __, model, __, trim = car_data
        else:
            year, __, make, __, model, __ = car_data
            trim = ''

        l.add_value('year', year)
        l.add_value('make', make)
        l.add_value('model', model)
        l.add_value('trim', trim)

        l.add_value('url', response.url)

        l.add_xpath('price', './/*[@class="single clearfix"]/span/text()')

        l.add_xpath('color', '//label[text()="Color:"]/following-sibling::span/text()')
        l.add_xpath('interior', '//label[text()="Interior:"]/following-sibling::span/text()')
        l.add_xpath('stock', '//label[text()="Stock#:"]/following-sibling::span/text()')
        l.add_xpath('engine', '//label[text()="Engine:"]/following-sibling::span/text()')
        l.add_xpath('vin', '//label[text()="VIN:"]/following-sibling::span/text()')
        l.add_xpath('transmission', '//label[text()="Transmission:"]/following-sibling::span/text()')
        l.add_xpath('odometer', '//label[text()="Odometer:"]/following-sibling::span/text()')
        l.add_xpath('body_style', '//label[text()="Body Style:"]/following-sibling::span/text()')

        return l.load_item()
<commit_msg>added testing<commit_after># -*- coding: utf-8 -*-
import urlparse

from scrapy import Spider
from scrapy.http import Request
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose
from w3lib.html import replace_escape_chars, remove_tags

from whitebearautos.items import WhitebearautosItem

class WhitebearautosspiderSpider(Spider):
    name = 'whitebearautos'
    allowed_domains = ['whitebearautos.com']
    start_urls = ['http://www.whitebearautos.com/inventory/view/Used/']

    def parse(self, response):
        """ Yields each car url and iterates over pages of results.

        @url http://www.whitebearautos.com/inventory/view/Used/
        @scrapes urls next_page_url

        """
        # process each car link
        urls = response.xpath('//h2/a/@href').extract()
        for url in urls:
            absolute_url = response.urljoin(url)
            request = Request(absolute_url, callback=self.parse_car)
            yield request

        # process next page
        next_page_url = response.xpath('.//*[@class="next"]/a/@href').extract_first()
        absolute_next_page_url = response.urljoin(next_page_url)
        request = Request(absolute_next_page_url)
        yield request

    def parse_car(self, response):
        """Returns scraped data from each individual car link.

        @url http://www.whitebearautos.com/Certified-2013-Mitsubishi-Lancer-ES-White-Bear-Lake-MN/vd/34254435
        @scrapes year make model trim price url color interior stock engine
        @scrapes vin transmission odometer body_style

        """
        l = ItemLoader(item=WhitebearautosItem(), response=response)
        l.default_output_processor = MapCompose(lambda v: v.strip(), replace_escape_chars)

        car_data = response.xpath('.//*[@class="cardata"]//text()').extract()
        if len(car_data) is 8:
            year, __, make, __, model, __, trim, __ = car_data
        elif len(car_data) is 7:
            year, __, make, __, model, __, trim = car_data
        else:
            year, __, make, __, model, __ = car_data
            trim = ''

        l.add_value('year', year)
        l.add_value('make', make)
        l.add_value('model', model)
        l.add_value('trim', trim)

        l.add_value('url', response.url)

        l.add_xpath('price', './/*[@class="single clearfix"]/span/text()')

        l.add_xpath('color', '//label[text()="Color:"]/following-sibling::span/text()')
        l.add_xpath('interior', '//label[text()="Interior:"]/following-sibling::span/text()')
        l.add_xpath('stock', '//label[text()="Stock#:"]/following-sibling::span/text()')
        l.add_xpath('engine', '//label[text()="Engine:"]/following-sibling::span/text()')
        l.add_xpath('vin', '//label[text()="VIN:"]/following-sibling::span/text()')
        l.add_xpath('transmission', '//label[text()="Transmission:"]/following-sibling::span/text()')
        l.add_xpath('odometer', '//label[text()="Odometer:"]/following-sibling::span/text()')
        l.add_xpath('body_style', '//label[text()="Body Style:"]/following-sibling::span/text()')

        return l.load_item()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import os
import random
try:
    import cPickle as pickle
except ImportError:
    import pickle
from lxml import etree
from operator import itemgetter

from odoo import api, fields, models, _
from odoo.exceptions import UserError
from odoo.release import version_info
from odoo.tools.safe_eval import safe_eval
from odoo.addons.anonymization.models.anonymization import group

WIZARD_ANONYMIZATION_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('unstable', 'Unstable')]

class IrModelFieldsAnonymizeWizard(models.TransientModel):
    _name = 'ir.model.fields.anonymize.wizard'

    name = fields.Char('File Name')
    summary = fields.Text(compute='_compute_summary')
    file_export = fields.Binary('Export')
    file_import = fields.Binary('Import',
        help="This is the file created by the anonymization process. It should have the '.pickle' extention.")
    state = fields.Selection(compute='_compute_state', string='Status', selection=WIZARD_ANONYMIZATION_STATES)
    msg = fields.Text('Message')

    @api.multi
    def _compute_summary(self):
        for anonymize_wizard in self:
            anonymize_wizard.summary = anonymize_wizard._get_summary_value()

    @api.multi
    def _compute_state(self):
        for anonymize_wizard in self:
            anonymize_wizard.state = anonymize_wizard._get_state_value()

    @api.model
    def _get_state_value(self):
        return self.env['ir.model.fields.anonymization']._get_global_state()

    @api.model
    def _get_summary_value(self):
        summary = u''
        for anon_field in self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')]):
            field = anon_field.field_id
            if field:
                values = {
                    'model_name': field.model_id.name,
                    'model_code': field.model_id.model,
                    'field_code': field.name,
                    'field_name': field.field_description,
                    'state': anon_field.state,
                }
                summary += u" * %(model_name)s (%(model_code)s) -> %(field_name)s (%(field_code)s): state: (%(state)s)\n" % values
            else:
                summary += u"* Missing local model (%s) and field (%s): state: (%s) \n" % (anon_field.model_name, anon_field.field_name, anon_field.state)
        return summary

    @api.model
    def default_get(self, fields_list):
        res = {}
        res['name'] = '.pickle'
        res['summary'] = self._get_summary_value()
        res['state'] = self._get_state_value()
        res['msg'] = _("Before executing the anonymization process, you should make a backup of your database.")
        return res

    @api.model
    def fields_view_get(self, view_id=None, view_type='form', toolbar=False, submenu=False):
        state = self.env['ir.model.fields.anonymization']._get_global_state()
        step = self.env.context.get('step', 'new_window')
        res = super(IrModelFieldsAnonymizeWizard, self).fields_view_get(view_id=view_id, view_type=view_type, toolbar=toolbar, submenu=submenu)
        eview = etree.fromstring(res['arch'])
        placeholder = eview.xpath("group[@name='placeholder1']")
        if len(placeholder):
            placeholder = placeholder[0]
            if step == 'new_window' and state == 'clear':
                # clicked in the menu and the fields are not anonymized: warn the admin that backuping the db is very important
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Warning'}))
                eview.remove(placeholder)
            elif step == 'new_window' and state == 'anonymized':
                # clicked in the menu and the fields are already anonymized
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('field', {'name': 'file_import', 'required': "1"}))
                placeholder.addnext(etree.Element('label', {'string': 'Anonymization file'}))
                eview.remove(placeholder)
            elif step == 'just_anonymized':
                # we just ran the anonymization process, we need the file export field
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('field', {'name': 'file_export'}))
                # we need to remove the button:
                buttons = eview.xpath("button")
                for button in buttons:
                    eview.remove(button)
                # and add a message:
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Result'}))
                # remove the placeholer:
                eview.remove(placeholder)
            elif step == 'just_desanonymized':
                # we just reversed the anonymization process, we don't need any field
                # we need to remove the button
                buttons = eview.xpath("button")
                for button in buttons:
                    eview.remove(button)
                # and add a message
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Result'}))
                # remove the placeholer:
                eview.remove(placeholder)
            else:
                raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                                  " while some fields are not anonymized. You should try to solve this problem before trying to do anything else."))
            res['arch'] = etree.tostring(eview)
        return res

    @api.multi
    def anonymize_database(self):
        """Sets the 'anonymized' state to defined fields"""
        self.ensure_one()

        # create a new history record:
        history = self.env['ir.model.fields.anonymization.history'].create({
            'date': fields.Datetime.now(),
            'state': 'started',
            'direction': 'clear -> anonymized'
        })

        # check that all the defined fields are in the 'clear' state
        state = self.env['ir.model.fields.anonymization']._get_global_state()
        error_type = _('Error !')
        if state == 'anonymized':
            raise UserError('%s: %s' % (error_type, _("The database is currently anonymized, you cannot anonymize it again.")))
        elif state == 'unstable':
            raise UserError('%s: %s' % (error_type, _("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                                                      " while some fields are not anonymized. You should try to solve this problem before trying to do anything.")))

        # do the anonymization:
        dirpath = os.environ.get('HOME') or os.getcwd()
        rel_filepath = 'field_anonymization_%s_%s.pickle' % (self.env.cr.dbname, history.id)
        abs_filepath = os.path.abspath(os.path.join(dirpath, rel_filepath))

        ano_fields = self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')])
        if not ano_fields:
            raise UserError('%s: %s' % (error_type, _("No fields are going to be anonymized.")))

        data = []

        for field in ano_fields:
            model_name = field.model_id.model
            field_name = field.field_id.name
            field_type = field.field_id.ttype
            table_name = self.env[model_name]._table

            # get the current value
            self.env.cr.execute("select id, %s from %s" % (field_name, table_name))
            for record in self.env.cr.dictfetchall():
                data.append({"model_id": model_name, "field_id": field_name, "id": record['id'], "value": record[field_name]})

                # anonymize the value:
                anonymized_value = None

                sid = str(record['id'])
                if field_type == 'char':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'selection':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'text':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'html':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'boolean':
                    anonymized_value = random.choice([True, False])
                elif field_type == 'date':
                    anonymized_value = '2011-11-11'
                elif field_type == 'datetime':
                    anonymized_value = '2011-11-11 11:11:11'
                elif field_type in ('float', 'monetary'):
                    anonymized_value = 0.0
                elif field_type == 'integer':
                    anonymized_value = 0
                elif field_type in ['binary', 'many2many', 'many2one', 'one2many', 'reference']:  # cannot anonymize these kind of fields
                    raise UserError('%s: %s' % (error_type, _("Cannot anonymize fields of these types: binary, many2many, many2one, one2many, reference.")))

                if anonymized_value is None:
                    raise UserError('%s: %s' % (error_type, _("Anonymized value can not be empty.")))

                sql = "update %(table)s set %(field)s = %%(anonymized_value)s where id = %%(id)s" % {
                    'table': table_name,
                    'field': field_name,
                }
                self.env.cr.execute(sql, {
                    'anonymized_value': anonymized_value,
                    'id': record['id']
                })

        # save pickle:
        fn = open(abs_filepath, 'w')
        pickle.dump(data, fn, pickle.HIGHEST_PROTOCOL)

        # update the anonymization fields:
        ano_fields.write({'state': 'anonymized'})

        # add a result message in the wizard:
        msgs = ["Anonymization successful.",
                "",
                "Donot forget to save the resulting file to a safe place because you will not be able to revert the anonymization without this file.",
                "",
                "This file is also stored in the %s directory. The absolute file path is: %s."
               ]
        msg = '\n'.join(msgs) % (dirpath, abs_filepath)

        fn = open(abs_filepath, 'r')

        self.write({
            'msg': msg,
            'file_export': base64.encodestring(fn.read()),
        })
        fn.close()

        # update the history record:
        history.write({
            'field_ids': [[6, 0, ano_fields.ids]],
            'msg': msg,
            'filepath': abs_filepath,
            'state': 'done',
        })

        return {
            'res_id': self.id,
            'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids,
            'view_type': 'form',
            "view_mode": 'form',
            'res_model': 'ir.model.fields.anonymize.wizard',
            'type': 'ir.actions.act_window',
            'context': {'step': 'just_anonymized'},
            'target': 'new'
        }

    @api.multi
    def reverse_anonymize_database(self):
        """Set the 'clear' state to defined fields"""
        self.ensure_one()
        IrModelFieldsAnonymization = self.env['ir.model.fields.anonymization']

        # check that all the defined fields are in the 'anonymized' state
        state = IrModelFieldsAnonymization._get_global_state()
        if state == 'clear':
            raise UserError(_("The database is not currently anonymized, you cannot reverse the anonymization."))
        elif state == 'unstable':
            raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                              " while some fields are not anonymized. You should try to solve this problem before trying to do anything."))

        if not self.file_import:
            raise UserError('%s: %s' % (_('Error !'), _("It is not possible to reverse the anonymization process without supplying the anonymization export file.")))

        # reverse the anonymization:
        # load the pickle file content into a data structure:
        data = pickle.loads(base64.decodestring(self.file_import))

        fixes = self.env['ir.model.fields.anonymization.migration.fix'].search_read([
            ('target_version', '=', '.'.join(map(str, version_info[:2])))
        ], ['model_name', 'field_name', 'query', 'query_type', 'sequence'])
        fixes = group(fixes, ('model_name', 'field_name'))

        for line in data:
            queries = []
            table_name = self.env[line['model_id']]._table if line['model_id'] in self.env else None

            # check if custom sql exists:
            key = (line['model_id'], line['field_id'])
            custom_updates = fixes.get(key)
            if custom_updates:
                custom_updates.sort(key=itemgetter('sequence'))
                queries = [(record['query'], record['query_type']) for record in custom_updates if record['query_type']]
            elif table_name:
                queries = [("update %(table)s set %(field)s = %%(value)s where id = %%(id)s" % {
                    'table': table_name,
                    'field': line['field_id'],
                }, 'sql')]

            for query in queries:
                if query[1] == 'sql':
                    self.env.cr.execute(query[0], {
                        'value': line['value'],
                        'id': line['id']
                    })
                elif query[1] == 'python':
                    safe_eval(query[0] % line)
                else:
                    raise Exception("Unknown query type '%s'. Valid types are: sql, python." % (query['query_type'], ))

            # update the anonymization fields:
            ano_fields = IrModelFieldsAnonymization.search([('state', '!=', 'not_existing')])
            ano_fields.write({'state': 'clear'})

            # add a result message in the wizard:
            self.msg = '\n'.join(["Successfully reversed the anonymization.", ""])

            # create a new history record:
            history = self.env['ir.model.fields.anonymization.history'].create({
                'date': fields.Datetime.now(),
                'field_ids': [[6, 0, ano_fields.ids]],
                'msg': self.msg,
                'filepath': False,
                'direction': 'anonymized -> clear',
                'state': 'done'
            })

            return {
                'res_id': self.id,
                'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids,
                'view_type': 'form',
                "view_mode": 'form',
                'res_model': 'ir.model.fields.anonymize.wizard',
                'type': 'ir.actions.act_window',
                'context': {'step': 'just_desanonymized'},
                'target': 'new'
            }
<commit_msg>[FIX] anonymization: fix port to new API<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import os
import random
try:
    import cPickle as pickle
except ImportError:
    import pickle
from lxml import etree
from operator import itemgetter

from odoo import api, fields, models, _
from odoo.exceptions import UserError
from odoo.release import version_info
from odoo.tools.safe_eval import safe_eval
from odoo.addons.anonymization.models.anonymization import group

WIZARD_ANONYMIZATION_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('unstable', 'Unstable')]

class IrModelFieldsAnonymizeWizard(models.TransientModel):
    _name = 'ir.model.fields.anonymize.wizard'

    name = fields.Char('File Name')
    summary = fields.Text(compute='_compute_summary')
    file_export = fields.Binary('Export')
    file_import = fields.Binary('Import',
        help="This is the file created by the anonymization process. It should have the '.pickle' extention.")
    state = fields.Selection(compute='_compute_state', string='Status', selection=WIZARD_ANONYMIZATION_STATES)
    msg = fields.Text('Message')

    @api.multi
    def _compute_summary(self):
        for anonymize_wizard in self:
            anonymize_wizard.summary = anonymize_wizard._get_summary_value()

    @api.multi
    def _compute_state(self):
        for anonymize_wizard in self:
            anonymize_wizard.state = anonymize_wizard._get_state_value()

    @api.model
    def _get_state_value(self):
        return self.env['ir.model.fields.anonymization']._get_global_state()

    @api.model
    def _get_summary_value(self):
        summary = u''
        for anon_field in self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')]):
            field = anon_field.field_id
            if field:
                values = {
                    'model_name': field.model_id.name,
                    'model_code': field.model_id.model,
                    'field_code': field.name,
                    'field_name': field.field_description,
                    'state': anon_field.state,
                }
                summary += u" * %(model_name)s (%(model_code)s) -> %(field_name)s (%(field_code)s): state: (%(state)s)\n" % values
            else:
                summary += u"* Missing local model (%s) and field (%s): state: (%s) \n" % (anon_field.model_name, anon_field.field_name, anon_field.state)
        return summary

    @api.model
    def default_get(self, fields_list):
        res = {}
        res['name'] = '.pickle'
        res['summary'] = self._get_summary_value()
        res['state'] = self._get_state_value()
        res['msg'] = _("Before executing the anonymization process, you should make a backup of your database.")
        return res

    @api.model
    def fields_view_get(self, view_id=None, view_type='form', toolbar=False, submenu=False):
        state = self.env['ir.model.fields.anonymization']._get_global_state()
        step = self.env.context.get('step', 'new_window')
        res = super(IrModelFieldsAnonymizeWizard, self).fields_view_get(view_id=view_id, view_type=view_type, toolbar=toolbar, submenu=submenu)
        eview = etree.fromstring(res['arch'])
        placeholder = eview.xpath("group[@name='placeholder1']")
        if len(placeholder):
            placeholder = placeholder[0]
            if step == 'new_window' and state == 'clear':
                # clicked in the menu and the fields are not anonymized: warn the admin that backuping the db is very important
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Warning'}))
                eview.remove(placeholder)
            elif step == 'new_window' and state == 'anonymized':
                # clicked in the menu and the fields are already anonymized
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('field', {'name': 'file_import', 'required': "1"}))
                placeholder.addnext(etree.Element('label', {'string': 'Anonymization file'}))
                eview.remove(placeholder)
            elif step == 'just_anonymized':
                # we just ran the anonymization process, we need the file export field
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('field', {'name': 'file_export'}))
                # we need to remove the button:
                buttons = eview.xpath("button")
                for button in buttons:
                    eview.remove(button)
                # and add a message:
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Result'}))
                # remove the placeholer:
                eview.remove(placeholder)
            elif step == 'just_desanonymized':
                # we just reversed the anonymization process, we don't need any field
                # we need to remove the button
                buttons = eview.xpath("button")
                for button in buttons:
                    eview.remove(button)
                # and add a message
                placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'}))
                placeholder.addnext(etree.Element('newline'))
                placeholder.addnext(etree.Element('label', {'string': 'Result'}))
                # remove the placeholer:
                eview.remove(placeholder)
            else:
                raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                                  " while some fields are not anonymized. You should try to solve this problem before trying to do anything else."))
            res['arch'] = etree.tostring(eview)
        return res

    @api.multi
    def anonymize_database(self):
        """Sets the 'anonymized' state to defined fields"""
        self.ensure_one()

        # create a new history record:
        history = self.env['ir.model.fields.anonymization.history'].create({
            'date': fields.Datetime.now(),
            'state': 'started',
            'direction': 'clear -> anonymized'
        })

        # check that all the defined fields are in the 'clear' state
        state = self.env['ir.model.fields.anonymization']._get_global_state()
        error_type = _('Error !')
        if state == 'anonymized':
            raise UserError('%s: %s' % (error_type, _("The database is currently anonymized, you cannot anonymize it again.")))
        elif state == 'unstable':
            raise UserError('%s: %s' % (error_type, _("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                                                      " while some fields are not anonymized. You should try to solve this problem before trying to do anything.")))

        # do the anonymization:
        dirpath = os.environ.get('HOME') or os.getcwd()
        rel_filepath = 'field_anonymization_%s_%s.pickle' % (self.env.cr.dbname, history.id)
        abs_filepath = os.path.abspath(os.path.join(dirpath, rel_filepath))

        ano_fields = self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')])
        if not ano_fields:
            raise UserError('%s: %s' % (error_type, _("No fields are going to be anonymized.")))

        data = []

        for field in ano_fields:
            model_name = field.model_id.model
            field_name = field.field_id.name
            field_type = field.field_id.ttype
            table_name = self.env[model_name]._table

            # get the current value
            self.env.cr.execute("select id, %s from %s" % (field_name, table_name))
            for record in self.env.cr.dictfetchall():
                data.append({"model_id": model_name, "field_id": field_name, "id": record['id'], "value": record[field_name]})

                # anonymize the value:
                anonymized_value = None

                sid = str(record['id'])
                if field_type == 'char':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'selection':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'text':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'html':
                    anonymized_value = 'xxx' + sid
                elif field_type == 'boolean':
                    anonymized_value = random.choice([True, False])
                elif field_type == 'date':
                    anonymized_value = '2011-11-11'
                elif field_type == 'datetime':
                    anonymized_value = '2011-11-11 11:11:11'
                elif field_type in ('float', 'monetary'):
                    anonymized_value = 0.0
                elif field_type == 'integer':
                    anonymized_value = 0
                elif field_type in ['binary', 'many2many', 'many2one', 'one2many', 'reference']:  # cannot anonymize these kind of fields
                    raise UserError('%s: %s' % (error_type, _("Cannot anonymize fields of these types: binary, many2many, many2one, one2many, reference.")))

                if anonymized_value is None:
                    raise UserError('%s: %s' % (error_type, _("Anonymized value can not be empty.")))

                sql = "update %(table)s set %(field)s = %%(anonymized_value)s where id = %%(id)s" % {
                    'table': table_name,
                    'field': field_name,
                }
                self.env.cr.execute(sql, {
                    'anonymized_value': anonymized_value,
                    'id': record['id']
                })

        # save pickle:
        fn = open(abs_filepath, 'w')
        pickle.dump(data, fn, pickle.HIGHEST_PROTOCOL)

        # update the anonymization fields:
        ano_fields.write({'state': 'anonymized'})

        # add a result message in the wizard:
        msgs = ["Anonymization successful.",
                "",
                "Donot forget to save the resulting file to a safe place because you will not be able to revert the anonymization without this file.",
                "",
                "This file is also stored in the %s directory. The absolute file path is: %s."
               ]
        msg = '\n'.join(msgs) % (dirpath, abs_filepath)

        fn = open(abs_filepath, 'r')

        self.write({
            'msg': msg,
            'file_export': base64.encodestring(fn.read()),
        })
        fn.close()

        # update the history record:
        history.write({
            'field_ids': [[6, 0, ano_fields.ids]],
            'msg': msg,
            'filepath': abs_filepath,
            'state': 'done',
        })

        return {
            'res_id': self.id,
            'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids,
            'view_type': 'form',
            "view_mode": 'form',
            'res_model': 'ir.model.fields.anonymize.wizard',
            'type': 'ir.actions.act_window',
            'context': {'step': 'just_anonymized'},
            'target': 'new'
        }

    @api.multi
    def reverse_anonymize_database(self):
        """Set the 'clear' state to defined fields"""
        self.ensure_one()
        IrModelFieldsAnonymization = self.env['ir.model.fields.anonymization']

        # check that all the defined fields are in the 'anonymized' state
        state = IrModelFieldsAnonymization._get_global_state()
        if state == 'clear':
            raise UserError(_("The database is not currently anonymized, you cannot reverse the anonymization."))
        elif state == 'unstable':
            raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                              " while some fields are not anonymized. You should try to solve this problem before trying to do anything."))

        if not self.file_import:
            raise UserError('%s: %s' % (_('Error !'), _("It is not possible to reverse the anonymization process without supplying the anonymization export file.")))

        # reverse the anonymization:
        # load the pickle file content into a data structure:
        data = pickle.loads(base64.decodestring(self.file_import))

        fixes = self.env['ir.model.fields.anonymization.migration.fix'].search_read([
            ('target_version', '=', '.'.join(map(str, version_info[:2])))
        ], ['model_name', 'field_name', 'query', 'query_type', 'sequence'])
        fixes = group(fixes, ('model_name', 'field_name'))

        for line in data:
            queries = []
            table_name = self.env[line['model_id']]._table if line['model_id'] in self.env else None

            # check if custom sql exists:
            key = (line['model_id'], line['field_id'])
            custom_updates = fixes.get(key)
            if custom_updates:
                custom_updates.sort(key=itemgetter('sequence'))
                queries = [(record['query'], record['query_type']) for record in custom_updates if record['query_type']]
            elif table_name:
                queries = [("update %(table)s set %(field)s = %%(value)s where id = %%(id)s" % {
                    'table': table_name,
                    'field': line['field_id'],
                }, 'sql')]

            for query in queries:
                if query[1] == 'sql':
                    self.env.cr.execute(query[0], {
                        'value': line['value'],
                        'id': line['id']
                    })
                elif query[1] == 'python':
                    safe_eval(query[0] % line)
                else:
                    raise Exception("Unknown query type '%s'. Valid types are: sql, python." % (query['query_type'], ))

        # update the anonymization fields:
        ano_fields = IrModelFieldsAnonymization.search([('state', '!=', 'not_existing')])
        ano_fields.write({'state': 'clear'})

        # add a result message in the wizard:
        self.msg = '\n'.join(["Successfully reversed the anonymization.", ""])

        # create a new history record:
        history = self.env['ir.model.fields.anonymization.history'].create({
            'date': fields.Datetime.now(),
            'field_ids': [[6, 0, ano_fields.ids]],
            'msg': self.msg,
            'filepath': False,
            'direction': 'anonymized -> clear',
            'state': 'done'
        })

        return {
            'res_id': self.id,
            'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids,
            'view_type': 'form',
            "view_mode": 'form',
            'res_model': 'ir.model.fields.anonymize.wizard',
            'type': 'ir.actions.act_window',
            'context': {'step': 'just_desanonymized'},
            'target': 'new'
        }
<|endoftext|>


<commit_before>#em_binner.py
#Will Barnes
#14 October 2015

#Class to construct emission measure (EM) distributions from temperature and density profiles. Incorporate effective temperature profiles due to non-equilibrium ionization.

#Import needed modules
import logging
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

class EM_Binner(object):

    def __init__(self,time,temp,density,loop_length,read_abundances=False,**kwargs):
        #set loop length as member variable
        self.loop_length = loop_length
        #Slice array and set member arrays
        self.time = time
        self.temp = temp
        self.density = density
        #set up logger
        self.logger = logging.getLogger(type(self).__name__)
        #Get abundances
        if(read_abundances):
            self.logger.warning("TODO: import abundances")
            pass

    def logT_bins(self,logT_a=4.0,logT_b=8.5,delta_logT=0.01):
        """Build temperature bins in log_10 for creating EM distribution"""

        self.delta_logT = delta_logT
        self.logT_EM = np.arange(logT_a,logT_b,delta_logT)
        #Add right edge for histogram bins
        self.logT_EM_histo_bins = np.append(self.logT_EM,self.logT_EM[-1]+self.delta_logT)

    def emission_measure_calc(self,n):
        """Calculate emission measure distribution"""

        return self.loop_length*n**2

    def coronal_limits(self,T):
        """Find limits of corona in log temperature space."""

        #Use EBTEL method for calculating coronal temperature bounds
        #logT_C_a = np.log10(2.0/3.0*T)
        logT_C_a = np.log10(8.0/9.0*T)
        logT_C_b = np.log10(10.0/9.0*T)
        return logT_C_a,logT_C_b

    def build_em_dist(self):
        """Create EM distribution from temperature arrays. Build for both T and T_eff"""

        try:
            self.logT_EM
        except AttributeError:
            self.logger.warning("Temperature bins not yet created. Building now with default values.")
            self.logT_bins()

        #Flattened EM and temp lists for easily building histograms
        self.em_flat = []
        self.logT_em_flat = []

        #Calculate time weights
        w_tau = np.gradient(self.time)/np.sum(np.gradient(self.time))

        #Loop over time
        for i in range(len(self.time)):
            #calculate coronal temperature bounds at time i
            logTa,logTb = self.coronal_limits(self.temp[i])
            #find coronal indices in logT
            iC = np.where((self.logT_EM >= logTa) & (self.logT_EM <= logTb))
            if len(iC) > 0:
                #append coronal temperatures to temperature list
                self.logT_em_flat.extend(self.logT_EM[iC[0]])
                #append emission measure weighted by timestep to emission measure list
                self.em_flat.extend(len(iC[0])*[w_tau[i]*self.emission_measure_calc(self.density[i])])
<commit_msg>added DEM calculation; EM_Binner class now calculates EM, DEM (coronal) matrices as is done in EBTEL<commit_after>#em_binner.py
#Will Barnes
#14 October 2015

#Class to construct emission measure (EM) distributions from temperature and density profiles. Incorporate effective temperature profiles due to non-equilibrium ionization.

#Import needed modules
import logging
import numpy as np
import matplotlib.pyplot as plt

class EM_Binner(object):

    def __init__(self,time,temp,density,loop_length,read_abundances=False,**kwargs):
        #set loop length as member variable
        self.loop_length = loop_length
        #Slice array and set member arrays
        self.time = time
        self.temp = temp
        self.density = density
        #set up logger
        self.logger = logging.getLogger(type(self).__name__)
        #Get abundances
        if(read_abundances):
            self.logger.warning("TODO: import abundances")
            pass

    def logT_bins(self,logT_a=4.0,logT_b=8.5,delta_logT=0.01):
        """Build temperature bins in log_10 for creating EM distribution"""

        self.delta_logT = delta_logT
        self.logT_EM = np.arange(logT_a,logT_b,delta_logT)
        #Add right edge for histogram bins
        self.logT_EM_histo_bins = np.append(self.logT_EM,self.logT_EM[-1]+self.delta_logT)

    def emission_measure_calc(self,n):
        """Calculate emission measure distribution"""

        return self.loop_length*n**2

    def differential_emission_measure_calc(self,n,tmin,tmax):
        """Calculate differential emission measure distribution"""

        em = self.emission_measure_calc(n)
        #EBTEL method for calculating coronal DEM
        jmax = (tmax - 4.0)*100.0
        jmin = (tmin - 4.0)*100.0
        delta_t = 1.0e+4*(10.0**((jmax + 0.5)/100.0) - 10.0**((jmin - 0.5)/100.0));
        return em/delta_t

    def coronal_limits(self,T):
        """Find limits of corona in log temperature space."""

        #Use EBTEL method for calculating coronal temperature bounds
        #logT_C_a = np.log10(2.0/3.0*T)
        logT_C_a = np.log10(8.0/9.0*T)
        logT_C_b = np.log10(10.0/9.0*T)
        return logT_C_a,logT_C_b

    def build_em_dist(self):
        """Create EM distribution from temperature arrays. Build for both T and T_eff"""

        try:
            self.logT_EM
        except AttributeError:
            self.logger.warning("Temperature bins not yet created. Building now with default values.")
            self.logT_bins()

        #allocate space for DEM, EM matrices
        self.em_mat = np.zeros(len(self.time),len(self.logT_EM))
        self.dem_mat = np.zeros(len(self.time),len(self.logT_EM))

        #Flattened EM and temp lists for easily building histograms
        self.em_flat = []
        self.logT_em_flat = []

        #Calculate time weights
        w_tau = np.gradient(self.time)/np.sum(np.gradient(self.time))

        #Loop over time
        for i in range(len(self.time)):
            #calculate coronal temperature bounds at time i
            logTa,logTb = self.coronal_limits(self.temp[i])
            #find coronal indices in logT
            iC = np.where((self.logT_EM >= logTa) & (self.logT_EM <= logTb))
            if len(iC) > 0:
                #add entries to dem,em matrices
                self.dem_mat[i,ic[0]] = self.differential_emission_measure_calc(self.density[i],logTa,logTb)
                self.em_mat[i,ic[0]] = self.emission_measure_calc(self.density[i])
                #append coronal temperatures to temperature list
                self.logT_em_flat.extend(self.logT_EM[iC[0]])
                #append emission measure weighted by timestep to emission measure list
                self.em_flat.extend(len(iC[0])*[w_tau[i]*self.emission_measure_calc(self.density[i])])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import markovify
import re
import spacy
import sys
import os
import argparse
import json
import errno
import random

nlp = spacy.load("en")

class POSifiedText(markovify.Text):
    def word_split(self, sentence):
        return ["::".join((word.orth_, word.pos_)) for word in nlp(sentence)]

    def word_join(self, words):
        sentence = " ".join(word.split("::")[0] for word in words)
        return sentence

parser = argparse.ArgumentParser()
parser.add_argument("config", help="The JSON config file to load.")
parser.add_argument("-r", "--refresh", help="Force a refresh of the corpora",
        action="store_true")

args = parser.parse_args()

config = {}
with open(args.config, 'r') as f:
    config = json.load(f)

bad_config = False
stanzas_missing = []

# Config handling and setup
def check_stanza(key, description):
    global bad_config, stanzas_missing
    if not key in config:
        bad_config = True
        stanzas_missing.append({'name':key, 'description':description})

check_stanza("main_corpus_source", "The path to the main corpus directory")
check_stanza("main_corpus_json", "The path to the main corpus JSON")
check_stanza("link_corpora_json_dir", "The path of the link corpora JSON files")
check_stanza("link_corpora", "The collection of link copora")
check_stanza("outputs", "The output for the raw text")

if bad_config:
    print("Error parsing config file!\n")
    print("The following stanzas were missing from the config file:\n")
    for st in stanzas_missing:
        print("\tstanza: '{0}'".format(st['name']))
        print("\t{0}\n".format(st['description']))
    sys.exit(1)

def load_corpus(json_file, fullpath):
    model = None
    if os.path.isfile(json_file) and not args.refresh:
        try:
            with open(json_file, 'r') as f:
                j = f.read()
            model = POSifiedText.from_json(j)
        except:
            print("Error parsing json file: {0}".format(json_file))
            print("Trying to load corpus, and JSON file invalid!")
            sys.exit(1)
    if model is None:
        # Okay, we load it from the directory
        for (dirpath, _, filenames) in os.walk(fullpath):
            for filename in filenames:
                with open(os.path.join(dirpath, filename)) as f:
                    m = POSifiedText(f, retain_original=False)
                if model:
                    model = markovify.combine(models=[model, m])
                else:
                    model = m

    return model

def save_corpus(json_file, model):
    j = model.to_json()
    with open(json_file, 'w') as f:
        f.write(j)

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass

# Load corpora
#--------------
main_model = load_corpus(config['main_corpus_json'],config['main_corpus_source'])
population = [ None ]
weights = [ config['main_weight'] ]
link_corpora = {}
for k in config['link_corpora']:
    n = k['name']
    jf = os.path.join(config['link_corpora_json_dir'], '{0}.json'.format(n))
    fp = k['source']
    model = load_corpus(jf, fp)
    link_corpora[n] = { 'model' : model, 'url' : k['url'] }
    population.append(n)
    weights.append(k['weight'])

# Save corpora
#--------------
save_corpus(config['main_corpus_json'], main_model)
for k in config['link_corpora']:
    n = k['name']
    jf = os.path.join(config['link_corpora_json_dir'], '{0}.json'.format(n))
    save_corpus(jf, link_corpora[n]['model'])

# Make the stuff
#----------------
for outs in config['outputs']:
    print("Processing output: {0}".format(outs['name']))
    mkdir_p(outs['out_dir'])
    num_sent = random.randint(
            outs['min_max']['sent_per_para'][0],
            outs['min_max']['sent_per_para'][1])
    num_para = random.randint(
            outs['min_max']['para_per_page'][0],
            outs['min_max']['para_per_page'][1])
    num_page = random.randint(
            outs['min_max']['pages'][0], outs['min_max']['pages'][1])
    page_count = 0
    while page_count < num_page:
        page_count += 1
        para_count = 0
        page_filename = os.path.join(outs['out_dir'],
                "page_{0}.html".format(page_count))
        with open(page_filename, 'w') as f:
            while para_count < num_para:
                para_count += 1
                f.write('
')
                sent_count = 0
                while sent_count < num_sent:
                    sent_count += 1
                    choice = random.choices(population, weights)[0]
                    if choice is None:
                        sent = main_model.make_sentence()
                        f.write('{0}. '.format(sent))
                    else:
                        sent = link_corpora[choice]['model'].make_sentence()
                        link = random.choice(link_corpora[choice]['url'])
                        f.write('{1}. '.format(link, sent))
                f.write('
\n\n')
<commit_msg>punctuation comes from markov chains<commit_after>#!/usr/bin/env python

import markovify
import re
import spacy
import sys
import os
import argparse
import json
import errno
import random

nlp = spacy.load("en")

class POSifiedText(markovify.Text):
    def word_split(self, sentence):
        return ["::".join((word.orth_, word.pos_)) for word in nlp(sentence)]

    def word_join(self, words):
        sentence = " ".join(word.split("::")[0] for word in words)
        return sentence

parser = argparse.ArgumentParser()
parser.add_argument("config", help="The JSON config file to load.")
parser.add_argument("-r", "--refresh", help="Force a refresh of the corpora",
        action="store_true")

args = parser.parse_args()

config = {}
with open(args.config, 'r') as f:
    config = json.load(f)

bad_config = False
stanzas_missing = []

# Config handling and setup
def check_stanza(key, description):
    global bad_config, stanzas_missing
    if not key in config:
        bad_config = True
        stanzas_missing.append({'name':key, 'description':description})

check_stanza("main_corpus_source", "The path to the main corpus directory")
check_stanza("main_corpus_json", "The path to the main corpus JSON")
check_stanza("link_corpora_json_dir", "The path of the link corpora JSON files")
check_stanza("link_corpora", "The collection of link copora")
check_stanza("outputs", "The output for the raw text")

if bad_config:
    print("Error parsing config file!\n")
    print("The following stanzas were missing from the config file:\n")
    for st in stanzas_missing:
        print("\tstanza: '{0}'".format(st['name']))
        print("\t{0}\n".format(st['description']))
    sys.exit(1)

def load_corpus(json_file, fullpath):
    model = None
    if os.path.isfile(json_file) and not args.refresh:
        try:
            with open(json_file, 'r') as f:
                j = f.read()
            model = POSifiedText.from_json(j)
        except:
            print("Error parsing json file: {0}".format(json_file))
            print("Trying to load corpus, and JSON file invalid!")
            sys.exit(1)
    if model is None:
        # Okay, we load it from the directory
        for (dirpath, _, filenames) in os.walk(fullpath):
            for filename in filenames:
                with open(os.path.join(dirpath, filename)) as f:
                    m = POSifiedText(f, retain_original=False)
                if model:
                    model = markovify.combine(models=[model, m])
                else:
                    model = m

    return model

def save_corpus(json_file, model):
    j = model.to_json()
    with open(json_file, 'w') as f:
        f.write(j)

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass

# Load corpora
#--------------
main_model = load_corpus(config['main_corpus_json'],config['main_corpus_source'])
population = [ None ]
weights = [ config['main_weight'] ]
link_corpora = {}
for k in config['link_corpora']:
    n = k['name']
    jf = os.path.join(config['link_corpora_json_dir'], '{0}.json'.format(n))
    fp = k['source']
    model = load_corpus(jf, fp)
    link_corpora[n] = { 'model' : model, 'url' : k['url'] }
    population.append(n)
    weights.append(k['weight'])

# Save corpora
#--------------
save_corpus(config['main_corpus_json'], main_model)
for k in config['link_corpora']:
    n = k['name']
    jf = os.path.join(config['link_corpora_json_dir'], '{0}.json'.format(n))
    save_corpus(jf, link_corpora[n]['model'])

# Make the stuff
#----------------
for outs in config['outputs']:
    print("Processing output: {0}".format(outs['name']))
    mkdir_p(outs['out_dir'])
    num_sent = random.randint(
            outs['min_max']['sent_per_para'][0],
            outs['min_max']['sent_per_para'][1])
    num_para = random.randint(
            outs['min_max']['para_per_page'][0],
            outs['min_max']['para_per_page'][1])
    num_page = random.randint(
            outs['min_max']['pages'][0], outs['min_max']['pages'][1])
    page_count = 0
    while page_count < num_page:
        page_count += 1
        para_count = 0
        page_filename = os.path.join(outs['out_dir'],
                "page_{0}.html".format(page_count))
        with open(page_filename, 'w') as f:
            while para_count < num_para:
                para_count += 1
                f.write('
')
                sent_count = 0
                while sent_count < num_sent:
                    sent_count += 1
                    choice = random.choices(population, weights)[0]
                    if choice is None:
                        sent = main_model.make_sentence()
                        f.write('{0} '.format(sent))
                    else:
                        sent = link_corpora[choice]['model'].make_sentence()
                        link = random.choice(link_corpora[choice]['url'])
                        f.write('{1} '.format(link, sent))
                f.write('
\n\n')
<|endoftext|>