<commit_before>from nose.tools import set_trace
from argparse import ArgumentParser
import logging
import sys
import time

from config import Configuration
from model import (
    _now,
    InvalidPost,
    Post,
    TIME_FORMAT,
)

class Script(object):
    pass

class BotScript(Script):
    """A script that operates on one or more bots."""

    log = logging.getLogger("Bot script")

    @classmethod
    def parser(cls):
        parser = ArgumentParser()
        parser.add_argument(
            '--config',
            help="Directory containing the botfriend database.",
            required=True,
        )
        parser.add_argument(
            '--bot',
            help='Operate on this bot.',
            nargs='*'
        )
        return parser

    def __init__(self):
        parser = self.parser()
        self.args = parser.parse_args()
        self.config = Configuration.from_directory(self.args.config, self.args.bot)

    def run(self):
        found = False
        for model in self.config.bots:
            try:
                found = True
                self.process_bot(model)
            except InvalidPost, e:
                # This _should_ crash the whole script -- we don't
                # want to commit invalid posts to the database.
                raise e
            except Exception, e:
                # Don't let a 'normal' error crash the whole script.
                model.implementation.log.error(e.message, exc_info=e)
        self.config._db.commit()
        if not found:
            self.log.error("Could not find any bot named %s in %s.",
                           self.args.bot, self.config.directory
            )

    def process_bot(self, bot_model):
        raise NotImplementedError()

class SingleBotScript(BotScript):
    """A script that _must_ be run against a single bot."""

    @classmethod
    def parser(cls):
        parser = ArgumentParser()
        parser.add_argument(
            '--config',
            help="Directory containing the botfriend database.",
            required=True,
        )
        parser.add_argument(
            '--bot',
            help='Operate on this bot.',
            required=True
        )
        return parser

class RepublicationScript(BotScript):
    """Attempt to publish already created posts that failed in their
    delivery.
    """
    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of posts to republish per bot.",
            type=int,
            default=1
        )
        return parser

    def process_bot(self, bot_model):
        undelivered = bot_model.undeliverable_posts.limit(self.args.limit)
        for post in undelivered:
            for publication in post.publications:
                if not publication.error:
                    continue
                # Find the publisher responsible for this
                matches  = [x for x in bot_model.implementation.publishers
                            if x.service == publication.service]
                if not matches:
                    # This bot doesn't use this publisher anymore.
                    continue
                [publisher] = matches
                bot_model.log.info(
                    "Attempting to republish to %s: %s" % (
                        publication.service,
                        post.content
                    )
                )
                publisher.publish(post, publication)
                if publication.error:
                    bot_model.log.info("Failure: %s" % publication.error)
                else:
                    bot_model.log.info("Success!")

class DashboardScript(BotScript):
    """Display the current status of one or more bots."""
    def process_bot(self, bot_model):

        now = _now()
        recent = bot_model.recent_posts().limit(1).all()
        if not recent:
            bot_model.log.info("Has never posted.")
        else:
            [recent] = recent
            bot_model.log.info("Most recent post: %s" % recent.content)
            for publication in recent.publications:
                if publication.error:
                    bot_model.log.info(
                        "%s ERROR: %s" % (publication.service, publication.error)
                    )
                else:
                    bot_model.log.info(
                        "%s posted %dm ago (%s)" % (
                            publication.service,
                            (now-publication.most_recent_attempt).total_seconds()/60,
                            publication.most_recent_attempt,
                        )
                    )

        def announce_list(count, content, what):
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d %s %s" % (count, item, what))
            bot_model.log.info("Next up: %s" % content)

        # Announce scheduled posts.
        scheduled = bot_model.scheduled
        next_post_time = bot_model.next_post_time
        if len(scheduled) > 0:
            first = scheduled[0]
            announce_list(len(scheduled), first.content, "scheduled")
            next_post_time = first.publish_at or bot_model.next_post_time

        # Announce backlog posts.
        try:
            backlog = bot_model.backlog
            if backlog:
                first = backlog[0]
                announce_list(len(backlog), first, "in backlog")
        except ValueError, e:
            pass

        if next_post_time:
            minutes = (next_post_time-now).total_seconds()/60
            if minutes < 0:
                when = "ASAP"
            else:
                when = "in %dm" % minutes
            bot_model.log.info("Next post %s" % when)
        else:
            bot_model.log.info("Next post not scheduled.")

class PostScript(BotScript):
    """Create a new post for one or all bots."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            '--dry-run',
            help="Show what would be posted, but don't post it or commit to the database.",
            action='store_true'
        )
        parser.add_argument(
            '--force',
            help="Post even if the scheduler would not normally post now.",
            action='store_true'
        )
        return parser

    def process_bot(self, bot_model):
        implementation = bot_model.implementation
        if self.args.force:
            bot_model.next_post_time = _now()
        posts = implementation.publishable_posts
        if self.args.dry_run:
            print bot_model.name
            for post in posts:
                print post.content
                print "-" * 80
                return

        # We're doing this for real.
        for post in posts:
            for publication in implementation.publish(post):
                publication.post.bot.log.info(publication.display())
        self.config._db.commit()

class StateShowScript(BotScript):
    """Show the internal state for a bot."""

    def process_bot(self, bot_model):
        last_update = bot_model.last_state_update_time
        print "State for %s (last update %s)" % (bot_model.name, last_update)
        print bot_model.state

class StateSetScript(SingleBotScript):
    """Set the internal state for a bot."""

    @classmethod
    def parser(cls):
        parser = SingleBotScript.parser()
        parser.add_argument(
            "--file",
            help="Load from this file instead of standard input.",
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        if self.args.file:
            fh = open(self.args.file)
        else:
            fh = sys.stdin
        data = fh.read().decode("utf8")
        bot_model.implementation.set_state(data)
        print bot_model.state

class StateRefreshScript(BotScript):
    """Refresh the internal state for a bot."""

    def process_bot(self, bot_model):
        bot_model.implementation.check_and_update_state(force=True)
        print "State for %s (last update %s)" % (
            bot_model.name, bot_model.last_state_update_time
        )
        print bot_model.state

class StressTestScript(BotScript):
    """Stress-test a bot's generative capabilities without posting anything."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            '--rounds',
            help="Run the bot's generator this many times. (Default is 10,000)",
            type=int,
            default=10000
        )
        return parser

    def process_bot(self, bot_model):
        bot_model.implementation.stress_test(self.args.rounds)

class PublisherTestScript(BotScript):
    """Verify  that a bot's publishers are functioning without posting anything."""

    def process_bot(self, bot_model):
        for publisher in bot_model.implementation.publishers:
            try:
                result = publisher.self_test() or ""
                print "GOOD %s %s %s" % (bot_model.name, publisher.service, result)
            except Exception, e:
                print "FAIL %s %s: %s" % (
                    bot_model.name, publisher.service, e
                )

class BacklogShowScript(BotScript):
    """Show the backlog posts for a bot."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of backlog posts shown.",
            type=int,
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        backlog = bot_model.implementation.backlog
        count = len(backlog)
        if self.args.limit:
            max_i = self.args.limit - 1
        else:
            max_i = None
        if count:
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d %s in backlog" % (count, item))
            for i, content in enumerate(backlog):
                bot_model.log.info(content)
                if i > max_i:
                    break
        else:
            bot_model.log.info("No backlog.")

class BacklogLoadScript(SingleBotScript):

    @classmethod
    def parser(cls):
        parser = SingleBotScript.parser()
        parser.add_argument(
            "--file",
            help="Load from this file instead of standard input.",
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        if self.args.file:
            fh = open(self.args.file)
        else:
            fh = sys.stdin
        # Process one backlog item per line of the input file.
        items = [x.strip().decode("utf8") for x in fh.readlines()]
        bot_model.implementation.extend_backlog(items)
        self.log.info("Appended %d items to backlog." % len(items))
        self.log.info("Backlog size now %d items" % len(bot_model.backlog))

class BacklogClearScript(SingleBotScript):

    def process_bot(self, bot_model):
        backlog = bot_model.backlog
        if backlog:
            bot_model.log.warn(
                "About to clear the backlog for %s.", bot_model.name
            )
            bot_model.log.warn(
                "Sleeping for 2 seconds to give you a chance to Ctrl-C."
            )
            time.sleep(2)
            bot_model.implementation.clear_backlog()

class ScheduledPostsShowScript(BotScript):
    """Show the scheduled posts for a bot."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of scheduled posts shown.",
            type=int,
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        scheduled = bot_model.scheduled
        count = len(scheduled)
        if self.args.limit:
            max_i = self.args.limit - 1
        else:
            max_i = None
        if count:
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d scheduled %s" % (count, item))
            for i, post in enumerate(scheduled):
                if max_i is not None and i > max_i:
                    break
                if post.publish_at:
                    when_post = post.publish_at.strftime(TIME_FORMAT)
                elif i == 0 and bot_model.next_post_time:
                    when_post = bot_model.next_post_time.strftime(TIME_FORMAT)
                else:
                    when_post = "No scheduled time"
                bot_model.log.info("%s | %s" % (when_post, post.content))
        else:
            bot_model.log.info("No scheduled posts.")

class SchedulePostsLoadScript(SingleBotScript):

    def process_bot(self, bot_model):
        bot.schedule_posts()

class ScheduledPostsClearScript(SingleBotScript):

    def process_bot(self, bot_model):
        scheduled = list(bot_model.scheduled)
        if scheduled:
            bot_model.log.warn(
                "About to remove all scheduled posts for %s.", bot_model.name
            )
            bot_model.log.warn(
                "Sleeping for 2 seconds to give you a chance to Ctrl-C."
            )
            time.sleep(2)
            for post in scheduled:
                self.config._db.delete(post)

        # Also reset the next post time.
        bot_model.next_post_time = bot_model.implementation.schedule_next_post([])
        if bot_model.next_post_time:
            bot_model.log.info("Next post at %s", bot_model.next_post_time)
        else:
            bot_model.log.info("Ready to post.")
<commit_msg>Fix process_bot implementation.<commit_after>from nose.tools import set_trace
from argparse import ArgumentParser
import logging
import sys
import time

from config import Configuration
from model import (
    _now,
    InvalidPost,
    Post,
    TIME_FORMAT,
)

class Script(object):
    pass

class BotScript(Script):
    """A script that operates on one or more bots."""

    log = logging.getLogger("Bot script")

    @classmethod
    def parser(cls):
        parser = ArgumentParser()
        parser.add_argument(
            '--config',
            help="Directory containing the botfriend database.",
            required=True,
        )
        parser.add_argument(
            '--bot',
            help='Operate on this bot.',
            nargs='*'
        )
        return parser

    def __init__(self):
        parser = self.parser()
        self.args = parser.parse_args()
        self.config = Configuration.from_directory(self.args.config, self.args.bot)

    def run(self):
        found = False
        for model in self.config.bots:
            try:
                found = True
                self.process_bot(model)
            except InvalidPost, e:
                # This _should_ crash the whole script -- we don't
                # want to commit invalid posts to the database.
                raise e
            except Exception, e:
                # Don't let a 'normal' error crash the whole script.
                model.implementation.log.error(e.message, exc_info=e)
        self.config._db.commit()
        if not found:
            self.log.error("Could not find any bot named %s in %s.",
                           self.args.bot, self.config.directory
            )

    def process_bot(self, bot_model):
        raise NotImplementedError()

class SingleBotScript(BotScript):
    """A script that _must_ be run against a single bot."""

    @classmethod
    def parser(cls):
        parser = ArgumentParser()
        parser.add_argument(
            '--config',
            help="Directory containing the botfriend database.",
            required=True,
        )
        parser.add_argument(
            '--bot',
            help='Operate on this bot.',
            required=True
        )
        return parser

class RepublicationScript(BotScript):
    """Attempt to publish already created posts that failed in their
    delivery.
    """
    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of posts to republish per bot.",
            type=int,
            default=1
        )
        return parser

    def process_bot(self, bot_model):
        undelivered = bot_model.undeliverable_posts.limit(self.args.limit)
        for post in undelivered:
            for publication in post.publications:
                if not publication.error:
                    continue
                # Find the publisher responsible for this
                matches  = [x for x in bot_model.implementation.publishers
                            if x.service == publication.service]
                if not matches:
                    # This bot doesn't use this publisher anymore.
                    continue
                [publisher] = matches
                bot_model.log.info(
                    "Attempting to republish to %s: %s" % (
                        publication.service,
                        post.content
                    )
                )
                publisher.publish(post, publication)
                if publication.error:
                    bot_model.log.info("Failure: %s" % publication.error)
                else:
                    bot_model.log.info("Success!")

class DashboardScript(BotScript):
    """Display the current status of one or more bots."""
    def process_bot(self, bot_model):

        now = _now()
        recent = bot_model.recent_posts().limit(1).all()
        if not recent:
            bot_model.log.info("Has never posted.")
        else:
            [recent] = recent
            bot_model.log.info("Most recent post: %s" % recent.content)
            for publication in recent.publications:
                if publication.error:
                    bot_model.log.info(
                        "%s ERROR: %s" % (publication.service, publication.error)
                    )
                else:
                    bot_model.log.info(
                        "%s posted %dm ago (%s)" % (
                            publication.service,
                            (now-publication.most_recent_attempt).total_seconds()/60,
                            publication.most_recent_attempt,
                        )
                    )

        def announce_list(count, content, what):
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d %s %s" % (count, item, what))
            bot_model.log.info("Next up: %s" % content)

        # Announce scheduled posts.
        scheduled = bot_model.scheduled
        next_post_time = bot_model.next_post_time
        if len(scheduled) > 0:
            first = scheduled[0]
            announce_list(len(scheduled), first.content, "scheduled")
            next_post_time = first.publish_at or bot_model.next_post_time

        # Announce backlog posts.
        try:
            backlog = bot_model.backlog
            if backlog:
                first = backlog[0]
                announce_list(len(backlog), first, "in backlog")
        except ValueError, e:
            pass

        if next_post_time:
            minutes = (next_post_time-now).total_seconds()/60
            if minutes < 0:
                when = "ASAP"
            else:
                when = "in %dm" % minutes
            bot_model.log.info("Next post %s" % when)
        else:
            bot_model.log.info("Next post not scheduled.")

class PostScript(BotScript):
    """Create a new post for one or all bots."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            '--dry-run',
            help="Show what would be posted, but don't post it or commit to the database.",
            action='store_true'
        )
        parser.add_argument(
            '--force',
            help="Post even if the scheduler would not normally post now.",
            action='store_true'
        )
        return parser

    def process_bot(self, bot_model):
        implementation = bot_model.implementation
        if self.args.force:
            bot_model.next_post_time = _now()
        posts = implementation.publishable_posts
        if self.args.dry_run:
            print bot_model.name
            for post in posts:
                print post.content
                print "-" * 80
                return

        # We're doing this for real.
        for post in posts:
            for publication in implementation.publish(post):
                publication.post.bot.log.info(publication.display())
        self.config._db.commit()

class StateShowScript(BotScript):
    """Show the internal state for a bot."""

    def process_bot(self, bot_model):
        last_update = bot_model.last_state_update_time
        print "State for %s (last update %s)" % (bot_model.name, last_update)
        print bot_model.state

class StateSetScript(SingleBotScript):
    """Set the internal state for a bot."""

    @classmethod
    def parser(cls):
        parser = SingleBotScript.parser()
        parser.add_argument(
            "--file",
            help="Load from this file instead of standard input.",
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        if self.args.file:
            fh = open(self.args.file)
        else:
            fh = sys.stdin
        data = fh.read().decode("utf8")
        bot_model.implementation.set_state(data)
        print bot_model.state

class StateRefreshScript(BotScript):
    """Refresh the internal state for a bot."""

    def process_bot(self, bot_model):
        bot_model.implementation.check_and_update_state(force=True)
        print "State for %s (last update %s)" % (
            bot_model.name, bot_model.last_state_update_time
        )
        print bot_model.state

class StressTestScript(BotScript):
    """Stress-test a bot's generative capabilities without posting anything."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            '--rounds',
            help="Run the bot's generator this many times. (Default is 10,000)",
            type=int,
            default=10000
        )
        return parser

    def process_bot(self, bot_model):
        bot_model.implementation.stress_test(self.args.rounds)

class PublisherTestScript(BotScript):
    """Verify  that a bot's publishers are functioning without posting anything."""

    def process_bot(self, bot_model):
        for publisher in bot_model.implementation.publishers:
            try:
                result = publisher.self_test() or ""
                print "GOOD %s %s %s" % (bot_model.name, publisher.service, result)
            except Exception, e:
                print "FAIL %s %s: %s" % (
                    bot_model.name, publisher.service, e
                )

class BacklogShowScript(BotScript):
    """Show the backlog posts for a bot."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of backlog posts shown.",
            type=int,
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        backlog = bot_model.implementation.backlog
        count = len(backlog)
        if self.args.limit:
            max_i = self.args.limit - 1
        else:
            max_i = None
        if count:
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d %s in backlog" % (count, item))
            for i, content in enumerate(backlog):
                bot_model.log.info(content)
                if i > max_i:
                    break
        else:
            bot_model.log.info("No backlog.")

class BacklogLoadScript(SingleBotScript):

    @classmethod
    def parser(cls):
        parser = SingleBotScript.parser()
        parser.add_argument(
            "--file",
            help="Load from this file instead of standard input.",
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        if self.args.file:
            fh = open(self.args.file)
        else:
            fh = sys.stdin
        # Process one backlog item per line of the input file.
        items = [x.strip().decode("utf8") for x in fh.readlines()]
        bot_model.implementation.extend_backlog(items)
        self.log.info("Appended %d items to backlog." % len(items))
        self.log.info("Backlog size now %d items" % len(bot_model.backlog))

class BacklogClearScript(SingleBotScript):

    def process_bot(self, bot_model):
        backlog = bot_model.backlog
        if backlog:
            bot_model.log.warn(
                "About to clear the backlog for %s.", bot_model.name
            )
            bot_model.log.warn(
                "Sleeping for 2 seconds to give you a chance to Ctrl-C."
            )
            time.sleep(2)
            bot_model.implementation.clear_backlog()

class ScheduledPostsShowScript(BotScript):
    """Show the scheduled posts for a bot."""

    @classmethod
    def parser(cls):
        parser = BotScript.parser()
        parser.add_argument(
            "--limit",
            help="Limit the number of scheduled posts shown.",
            type=int,
            default=None
        )
        return parser

    def process_bot(self, bot_model):
        scheduled = bot_model.scheduled
        count = len(scheduled)
        if self.args.limit:
            max_i = self.args.limit - 1
        else:
            max_i = None
        if count:
            if count == 1:
                item = "post"
            else:
                item = "posts"
            bot_model.log.info("%d scheduled %s" % (count, item))
            for i, post in enumerate(scheduled):
                if max_i is not None and i > max_i:
                    break
                if post.publish_at:
                    when_post = post.publish_at.strftime(TIME_FORMAT)
                elif i == 0 and bot_model.next_post_time:
                    when_post = bot_model.next_post_time.strftime(TIME_FORMAT)
                else:
                    when_post = "No scheduled time"
                bot_model.log.info("%s | %s" % (when_post, post.content))
        else:
            bot_model.log.info("No scheduled posts.")

class SchedulePostsLoadScript(SingleBotScript):

    def process_bot(self, bot_model):
        bot_model.implementation.schedule_posts()

class ScheduledPostsClearScript(SingleBotScript):

    def process_bot(self, bot_model):
        scheduled = list(bot_model.scheduled)
        if scheduled:
            bot_model.log.warn(
                "About to remove all scheduled posts for %s.", bot_model.name
            )
            bot_model.log.warn(
                "Sleeping for 2 seconds to give you a chance to Ctrl-C."
            )
            time.sleep(2)
            for post in scheduled:
                self.config._db.delete(post)

        # Also reset the next post time.
        bot_model.next_post_time = bot_model.implementation.schedule_next_post([])
        if bot_model.next_post_time:
            bot_model.log.info("Next post at %s", bot_model.next_post_time)
        else:
            bot_model.log.info("Ready to post.")
<|endoftext|>


<commit_before># Copyright (C) 2019  Renato Lima - Akretion <renato.lima@akretion.com.br>
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import api, fields, models
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT

from ..constants.fiscal import (
    FINAL_CUSTOMER,
    FINAL_CUSTOMER_YES,
    FISCAL_COMMENT_DOCUMENT,
    NFE_IND_PRES,
    NFE_IND_PRES_DEFAULT,
)

class FiscalDocumentMixin(models.AbstractModel):
    _name = "l10n_br_fiscal.document.mixin"
    _inherit = "l10n_br_fiscal.document.mixin.methods"
    _description = "Document Fiscal Mixin"

    def _date_server_format(self):
        return fields.Datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)

    @api.model
    def _default_operation(self):
        return False

    @api.model
    def _operation_domain(self):
        domain = (
            "[('state', '=', 'approved'),"
            "'|',"
            "('company_id', '=', %s),"
            "('company_id', '=', False),"
        ) % (self.env.company.id,)
        return domain

    fiscal_operation_id = fields.Many2one(
        comodel_name="l10n_br_fiscal.operation",
        string="Operation",
        domain=lambda self: self._operation_domain(),
        default=_default_operation,
    )

    #
    # Company and Partner are defined here to avoid warnings on runbot
    #
    company_id = fields.Many2one(
        comodel_name="res.company",
        string="Company",
    )

    partner_id = fields.Many2one(
        comodel_name="res.partner",
        index=True,
    )

    fiscal_operation_type = fields.Selection(
        related="fiscal_operation_id.fiscal_operation_type",
        string="Fiscal Operation Type",
        readonly=True,
    )

    ind_pres = fields.Selection(
        selection=NFE_IND_PRES,
        string="Buyer Presence",
        default=NFE_IND_PRES_DEFAULT,
    )

    comment_ids = fields.Many2many(
        comodel_name="l10n_br_fiscal.comment",
        relation="l10n_br_fiscal_document_mixin_comment_rel",
        column1="document_mixin_id",
        column2="comment_id",
        string="Comments",
        domain=[("object", "=", FISCAL_COMMENT_DOCUMENT)],
    )

    fiscal_additional_data = fields.Text()

    manual_fiscal_additional_data = fields.Char(
        help="Fiscal Additional data manually entered by user",
    )

    customer_additional_data = fields.Text()

    manual_customer_additional_data = fields.Char(
        help="Customer Additional data manually entered by user",
    )

    ind_final = fields.Selection(
        selection=FINAL_CUSTOMER,
        string="Final Consumption Operation",
        default=FINAL_CUSTOMER_YES,
    )

    currency_id = fields.Many2one(
        comodel_name="res.currency",
        default=lambda self: self.env.company.currency_id,
        store=True,
        readonly=True,
    )

    amount_price_gross = fields.Monetary(
        compute="_compute_amount",
        store=True,
        string="Amount Gross",
        help="Amount without discount.",
    )

    amount_untaxed = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_icms_base = fields.Monetary(
        string="ICMS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_value = fields.Monetary(
        string="ICMS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsst_base = fields.Monetary(
        string="ICMS ST Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsst_value = fields.Monetary(
        string="ICMS ST Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmssn_credit_value = fields.Monetary(
        string="ICMSSN Credit Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsfcp_base = fields.Monetary(
        string="ICMS FCP Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsfcp_value = fields.Monetary(
        string="ICMS FCP Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_destination_value = fields.Monetary(
        string="ICMS Destination Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_origin_value = fields.Monetary(
        string="ICMS Origin Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ipi_base = fields.Monetary(
        string="IPI Base",
        compute="_compute_amount",
        store=True,
    )

    amount_ipi_value = fields.Monetary(
        string="IPI Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_base = fields.Monetary(
        string="II Base",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_value = fields.Monetary(
        string="II Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_customhouse_charges = fields.Monetary(
        string="Customhouse Charges",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_base = fields.Monetary(
        string="PIS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_value = fields.Monetary(
        string="PIS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_wh_base = fields.Monetary(
        string="PIS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_wh_value = fields.Monetary(
        string="PIS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_base = fields.Monetary(
        string="COFINS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_value = fields.Monetary(
        string="COFINS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_wh_base = fields.Monetary(
        string="COFINS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_wh_value = fields.Monetary(
        string="COFINS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_base = fields.Monetary(
        string="ISSQN Base",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_value = fields.Monetary(
        string="ISSQN Value",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_wh_base = fields.Monetary(
        string="ISSQN Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_wh_value = fields.Monetary(
        string="ISSQN Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_base = fields.Monetary(
        string="CSLL Base",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_value = fields.Monetary(
        string="CSLL Value",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_wh_base = fields.Monetary(
        string="CSLL Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_wh_value = fields.Monetary(
        string="CSLL Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_base = fields.Monetary(
        string="IRPJ Base",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_value = fields.Monetary(
        string="IRPJ Value",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_wh_base = fields.Monetary(
        string="IRPJ Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_wh_value = fields.Monetary(
        string="IRPJ Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_base = fields.Monetary(
        string="INSS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_value = fields.Monetary(
        string="INSS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_wh_base = fields.Monetary(
        string="INSS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_wh_value = fields.Monetary(
        string="INSS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_estimate_tax = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_tax = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_total = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_tax_withholding = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_financial_total = fields.Monetary(
        string="Amount Financial",
        compute="_compute_amount",
        store=True,
    )

    amount_discount_value = fields.Monetary(
        string="Amount Discount",
        compute="_compute_amount",
        store=True,
    )

    amount_financial_total_gross = fields.Monetary(
        string="Amount Financial Gross",
        compute="_compute_amount",
        store=True,
    )

    amount_financial_discount_value = fields.Monetary(
        string="Financial Discount Value",
        compute="_compute_amount",
        store=True,
    )

    amount_insurance_value = fields.Monetary(
        string="Insurance Value",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_insurance",
    )

    amount_other_value = fields.Monetary(
        string="Other Costs",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_other",
    )

    amount_freight_value = fields.Monetary(
        string="Freight Value",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_freight",
    )

    # Usado para tornar Somente Leitura os campos totais dos custos
    # de entrega quando a definição for por Linha
    delivery_costs = fields.Selection(
        related="company_id.delivery_costs",
    )

    force_compute_delivery_costs_by_total = fields.Boolean(
        "Force compute delivery costs by total", default=False
    )
<commit_msg>fixup! [REF] attribute-string-redundant l10n_br_fiscal<commit_after># Copyright (C) 2019  Renato Lima - Akretion <renato.lima@akretion.com.br>
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import api, fields, models
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT

from ..constants.fiscal import (
    FINAL_CUSTOMER,
    FINAL_CUSTOMER_YES,
    FISCAL_COMMENT_DOCUMENT,
    NFE_IND_PRES,
    NFE_IND_PRES_DEFAULT,
)

class FiscalDocumentMixin(models.AbstractModel):
    _name = "l10n_br_fiscal.document.mixin"
    _inherit = "l10n_br_fiscal.document.mixin.methods"
    _description = "Document Fiscal Mixin"

    def _date_server_format(self):
        return fields.Datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)

    @api.model
    def _default_operation(self):
        return False

    @api.model
    def _operation_domain(self):
        domain = (
            "[('state', '=', 'approved'),"
            "'|',"
            "('company_id', '=', %s),"
            "('company_id', '=', False),"
        ) % (self.env.company.id,)
        return domain

    fiscal_operation_id = fields.Many2one(
        comodel_name="l10n_br_fiscal.operation",
        string="Operation",
        domain=lambda self: self._operation_domain(),
        default=_default_operation,
    )

    #
    # Company and Partner are defined here to avoid warnings on runbot
    #
    company_id = fields.Many2one(
        comodel_name="res.company",
        string="Company",
    )

    partner_id = fields.Many2one(
        comodel_name="res.partner",
        index=True,
    )

    fiscal_operation_type = fields.Selection(
        related="fiscal_operation_id.fiscal_operation_type",
        string="Fiscal Operation Type",
        readonly=True,
    )

    ind_pres = fields.Selection(
        selection=NFE_IND_PRES,
        string="Buyer Presence",
        default=NFE_IND_PRES_DEFAULT,
    )

    comment_ids = fields.Many2many(
        comodel_name="l10n_br_fiscal.comment",
        relation="l10n_br_fiscal_document_mixin_comment_rel",
        column1="document_mixin_id",
        column2="comment_id",
        string="Comments",
        domain=[("object", "=", FISCAL_COMMENT_DOCUMENT)],
    )

    fiscal_additional_data = fields.Text()

    manual_fiscal_additional_data = fields.Char(
        help="Fiscal Additional data manually entered by user",
    )

    customer_additional_data = fields.Text()

    manual_customer_additional_data = fields.Char(
        help="Customer Additional data manually entered by user",
    )

    ind_final = fields.Selection(
        selection=FINAL_CUSTOMER,
        string="Final Consumption Operation",
        default=FINAL_CUSTOMER_YES,
    )

    currency_id = fields.Many2one(
        comodel_name="res.currency",
        default=lambda self: self.env.company.currency_id,
        store=True,
        readonly=True,
    )

    amount_price_gross = fields.Monetary(
        compute="_compute_amount",
        store=True,
        string="Amount Gross",
        help="Amount without discount.",
    )

    amount_untaxed = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_icms_base = fields.Monetary(
        string="ICMS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_value = fields.Monetary(
        string="ICMS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsst_base = fields.Monetary(
        string="ICMS ST Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsst_value = fields.Monetary(
        string="ICMS ST Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmssn_credit_value = fields.Monetary(
        string="ICMSSN Credit Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsfcp_base = fields.Monetary(
        string="ICMS FCP Base",
        compute="_compute_amount",
        store=True,
    )

    amount_icmsfcp_value = fields.Monetary(
        string="ICMS FCP Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_destination_value = fields.Monetary(
        string="ICMS Destination Value",
        compute="_compute_amount",
        store=True,
    )

    amount_icms_origin_value = fields.Monetary(
        string="ICMS Origin Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ipi_base = fields.Monetary(
        string="IPI Base",
        compute="_compute_amount",
        store=True,
    )

    amount_ipi_value = fields.Monetary(
        string="IPI Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_base = fields.Monetary(
        string="II Base",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_value = fields.Monetary(
        string="II Value",
        compute="_compute_amount",
        store=True,
    )

    amount_ii_customhouse_charges = fields.Monetary(
        string="Customhouse Charges",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_base = fields.Monetary(
        string="PIS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_value = fields.Monetary(
        string="PIS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_wh_base = fields.Monetary(
        string="PIS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_pis_wh_value = fields.Monetary(
        string="PIS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_base = fields.Monetary(
        string="COFINS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_value = fields.Monetary(
        string="COFINS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_wh_base = fields.Monetary(
        string="COFINS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_cofins_wh_value = fields.Monetary(
        string="COFINS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_base = fields.Monetary(
        string="ISSQN Base",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_value = fields.Monetary(
        string="ISSQN Value",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_wh_base = fields.Monetary(
        string="ISSQN Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_issqn_wh_value = fields.Monetary(
        string="ISSQN Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_base = fields.Monetary(
        string="CSLL Base",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_value = fields.Monetary(
        string="CSLL Value",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_wh_base = fields.Monetary(
        string="CSLL Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_csll_wh_value = fields.Monetary(
        string="CSLL Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_base = fields.Monetary(
        string="IRPJ Base",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_value = fields.Monetary(
        string="IRPJ Value",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_wh_base = fields.Monetary(
        string="IRPJ Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_irpj_wh_value = fields.Monetary(
        string="IRPJ Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_base = fields.Monetary(
        string="INSS Base",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_value = fields.Monetary(
        string="INSS Value",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_wh_base = fields.Monetary(
        string="INSS Ret Base",
        compute="_compute_amount",
        store=True,
    )

    amount_inss_wh_value = fields.Monetary(
        string="INSS Ret Value",
        compute="_compute_amount",
        store=True,
    )

    amount_estimate_tax = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_tax = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_total = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_tax_withholding = fields.Monetary(
        compute="_compute_amount",
        store=True,
    )

    amount_financial_total = fields.Monetary(
        string="Amount Financial",
        compute="_compute_amount",
        store=True,
    )

    amount_discount_value = fields.Monetary(
        string="Amount Discount",
        compute="_compute_amount",
        store=True,
    )

    amount_financial_total_gross = fields.Monetary(
        string="Amount Financial Gross",
        compute="_compute_amount",
        store=True,
    )

    amount_financial_discount_value = fields.Monetary(
        string="Financial Discount Value",
        compute="_compute_amount",
        store=True,
    )

    amount_insurance_value = fields.Monetary(
        string="Insurance Value",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_insurance",
    )

    amount_other_value = fields.Monetary(
        string="Other Costs",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_other",
    )

    amount_freight_value = fields.Monetary(
        string="Freight Value",
        compute="_compute_amount",
        store=True,
        inverse="_inverse_amount_freight",
    )

    # Usado para tornar Somente Leitura os campos totais dos custos
    # de entrega quando a definição for por Linha
    delivery_costs = fields.Selection(
        related="company_id.delivery_costs",
    )

    force_compute_delivery_costs_by_total = fields.Boolean(default=False)
<|endoftext|>


<commit_before>"""
Django models for site configurations.
"""

import collections
from logging import getLogger
import os

from django.conf import settings
from django.core.files.storage import get_storage_class
from django.contrib.sites.models import Site
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.core.files.storage import get_storage_class
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from jsonfield.fields import JSONField
from model_utils.models import TimeStampedModel

from .exceptions import TahoeConfigurationException

logger = getLogger(__name__)  # pylint: disable=invalid-name

def get_initial_sass_variables():
    """
    Proxy to `utils.get_initial_sass_variables` to avoid test-time Django errors.

    # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
    """
    from openedx.core.djangoapps.appsembler.sites import utils
    return utils.get_initial_sass_variables()

def get_initial_page_elements():
    """
    Proxy to `utils.get_initial_page_elements` to avoid test-time Django errors.

    # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
    """
    from openedx.core.djangoapps.appsembler.sites import utils
    return utils.get_initial_page_elements()

@python_2_unicode_compatible
class SiteConfiguration(models.Model):
    """
    Model for storing site configuration. These configuration override OpenEdx configurations and settings.
    e.g. You can override site name, logo image, favicon etc. using site configuration.

    Fields:
        site (OneToOneField): one to one field relating each configuration to a single site
        site_values (JSONField):  json field to store configurations for a site

    .. no_pii:
    """
    site = models.OneToOneField(Site, related_name='configuration', on_delete=models.CASCADE)
    enabled = models.BooleanField(default=False, verbose_name=u"Enabled")
    site_values = JSONField(
        null=False,
        blank=True,
        # The actual default value is determined by calling the given callable.
        # Therefore, the default here is just {}, since that is the result of
        # calling `dict`.
        default=dict,
        load_kwargs={'object_pairs_hook': collections.OrderedDict}
    )
    sass_variables = JSONField(blank=True, default=get_initial_sass_variables)
    page_elements = JSONField(blank=True, default=get_initial_page_elements)

    def __str__(self):
        return u"<SiteConfiguration: {site} >".format(site=self.site)  # xss-lint: disable=python-wrap-html

    def __repr__(self):
        return self.__str__()

    def save(self, **kwargs):
        # When creating a new object, save default microsite values. Not implemented as a default method on the field
        # because it depends on other fields that should be already filled.
        if not self.id:
            self.site_values = self.get_initial_microsite_values()

        # fix for a bug with some pages requiring uppercase platform_name variable
        self.site_values['PLATFORM_NAME'] = self.site_values.get('platform_name', '')

        # Set the default language code for new sites if missing
        # TODO: Move it to somewhere else like in AMC
        self.site_values['LANGUAGE_CODE'] = self.site_values.get('LANGUAGE_CODE', 'en')

        super(SiteConfiguration, self).save(**kwargs)

        # recompile SASS on every save
        self.compile_microsite_sass()
        return self

    def get_value(self, name, default=None):
        """
        Return Configuration value for the key specified as name argument.

        Function logs a message if configuration is not enabled or if there is an error retrieving a key.

        Args:
            name (str): Name of the key for which to return configuration value.
            default: default value tp return if key is not found in the configuration

        Returns:
            Configuration value for the given key or returns `None` if configuration is not enabled.
        """
        if self.enabled:
            try:
                return self.site_values.get(name, default)
            except AttributeError as error:
                logger.exception(u'Invalid JSON data. \n [%s]', error)
        else:
            logger.info(u"Site Configuration is not enabled for site (%s).", self.site)

        return default

    @classmethod
    def get_configuration_for_org(cls, org, select_related=None):
        """
        This returns a SiteConfiguration object which has an org_filter that matches
        the supplied org

        Args:
            org (str): Org to use to filter SiteConfigurations
            select_related (list or None): A list of values to pass as arguments to select_related
        """
        query = cls.objects.filter(site_values__contains=org, enabled=True).defer('page_elements', 'sass_variables').all()
        if select_related is not None:
            query = query.select_related(*select_related)
        for configuration in query:
            course_org_filter = configuration.get_value('course_org_filter', [])
            # The value of 'course_org_filter' can be configured as a string representing
            # a single organization or a list of strings representing multiple organizations.
            if not isinstance(course_org_filter, list):
                course_org_filter = [course_org_filter]
            if org in course_org_filter:
                return configuration
        return None

    @classmethod
    def get_value_for_org(cls, org, name, default=None):
        """
        This returns site configuration value which has an org_filter that matches
        what is passed in,

        Args:
            org (str): Course ord filter, this value will be used to filter out the correct site configuration.
            name (str): Name of the key for which to return configuration value.
            default: default value tp return if key is not found in the configuration

        Returns:
            Configuration value for the given key.
        """
        configuration = cls.get_configuration_for_org(org)
        if configuration is None:
            return default
        else:
            return configuration.get_value(name, default)

    @classmethod
    def get_all_orgs(cls):
        """
        This returns all of the orgs that are considered in site configurations, This can be used,
        for example, to do filtering.

        Returns:
            A set of all organizations present in site configuration.
        """
        org_filter_set = set()

        for configuration in cls.objects.filter(site_values__contains='course_org_filter', enabled=True).defer('page_elements', 'sass_variables').all():
            course_org_filter = configuration.get_value('course_org_filter', [])
            if not isinstance(course_org_filter, list):
                course_org_filter = [course_org_filter]
            org_filter_set.update(course_org_filter)
        return org_filter_set

    @classmethod
    def has_org(cls, org):
        """
        Check if the given organization is present in any of the site configuration.

        Returns:
            True if given organization is present in site configurations otherwise False.
        """
        return org in cls.get_all_orgs()

    def delete(self, using=None):
        self.delete_css_override()
        super(SiteConfiguration, self).delete(using=using)

    def compile_microsite_sass(self):
        # Importing `compile_sass` to avoid test-time Django errors.
        # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
        from openedx.core.djangoapps.appsembler.sites.utils import compile_sass
        css_output = compile_sass('main.scss', custom_branding=self._sass_var_override)
        file_name = self.get_value('css_overrides_file')

        if not file_name:
            if settings.TAHOE_SILENT_MISSING_CSS_CONFIG:
                return  # Silent the exception below on during testing
            else:
                raise TahoeConfigurationException(
                    'Missing `css_overrides_file` from SiteConfiguration for `{site}` config_id=`{id}`'.format(
                        site=self.site.domain,
                        id=self.id,
                    )
                )

        if settings.USE_S3_FOR_CUSTOMER_THEMES:
            storage_class = get_storage_class(settings.DEFAULT_FILE_STORAGE)
            storage = storage_class(
                location="customer_themes",
            )
            with storage.open(file_name, 'w') as f:
                f.write(css_output)
        else:
            theme_folder = os.path.join(settings.COMPREHENSIVE_THEME_DIRS[0], 'customer_themes')
            theme_file = os.path.join(theme_folder, file_name)
            with open(theme_file, 'w') as f:
                f.write(css_output)

    def get_css_url(self):
        if settings.USE_S3_FOR_CUSTOMER_THEMES:
            kwargs = {
                'location': "customer_themes",
            }
            storage = get_storage_class()(**kwargs)
            return storage.url(self.get_value('css_overrides_file'))
        else:
            return static("customer_themes/{}".format(self.get_value('css_overrides_file')))

    def set_sass_variables(self, entries):
        """
        Accepts a dict with the shape { var_name: value } and sets the SASS variables
        """
        for index, entry in enumerate(self.sass_variables):
            var_name = entry[0]
            if var_name in entries:
                new_value = (var_name, [entries[var_name], entries[var_name]])
                self.sass_variables[index] = new_value

    def delete_css_override(self):
        css_file = self.values.get('css_overrides_file')
        if css_file:
            try:
                if settings.USE_S3_FOR_CUSTOMER_THEMES:
                    kwargs = {
                        'location': "customer_themes",
                    }
                    storage = get_storage_class()(**kwargs)
                    storage.delete(self.get_value('css_overrides_file'))
                else:
                    os.remove(os.path.join(settings.COMPREHENSIVE_THEME_DIRS[0], css_file))
            except OSError:
                logger.warning("Can't delete CSS file {}".format(css_file))

    def _formatted_sass_variables(self):
        return " ".join(["{}: {};".format(var, val[0]) for var, val in self.sass_variables])

    def _sass_var_override(self, path):
        if 'branding-basics' in path:
            return [(path, self._formatted_sass_variables())]
        if 'customer-sass-input' in path:
            return [(path, self.values.get('customer_sass_input', ''))]
        return None

    def get_initial_microsite_values(self):
        domain_without_port_number = self.site.domain.split(':')[0]
        return {
            'platform_name': self.site.name,
            'css_overrides_file': "{}.css".format(domain_without_port_number),
            'ENABLE_COMBINED_LOGIN_REGISTRATION': True,
        }

def save_siteconfig_without_historical_record(siteconfig, *args, **kwargs):
    """
    Save model without saving a historical record

    Make sure you know what you're doing before you use this method.

    Note: this method is copied verbatim from django-simple-history.
    """
    siteconfig.skip_history_when_saving = True
    try:
        ret = siteconfig.save(*args, **kwargs)
    finally:
        del siteconfig.skip_history_when_saving
    return ret

@python_2_unicode_compatible
class SiteConfigurationHistory(TimeStampedModel):
    """
    This is an archive table for SiteConfiguration, so that we can maintain a history of
    changes. Note that the site field is not unique in this model, compared to SiteConfiguration.

    Fields:
        site (ForeignKey): foreign-key to django Site
        site_values (JSONField): json field to store configurations for a site

    .. no_pii:
    """
    site = models.ForeignKey(Site, related_name='configuration_histories', on_delete=models.CASCADE)
    enabled = models.BooleanField(default=False, verbose_name=u"Enabled")
    site_values = JSONField(
        null=False,
        blank=True,
        load_kwargs={'object_pairs_hook': collections.OrderedDict}
    )

    class Meta:
        get_latest_by = 'modified'
        ordering = ('-modified', '-created',)

    def __str__(self):
        # pylint: disable=line-too-long
        return u"<SiteConfigurationHistory: {site}, Last Modified: {modified} >".format(  # xss-lint: disable=python-wrap-html
            modified=self.modified,
            site=self.site,
        )

    def __repr__(self):
        return self.__str__()

@receiver(post_save, sender=SiteConfiguration)
def update_site_configuration_history(sender, instance, created, **kwargs):  # pylint: disable=unused-argument
    """
    Add site configuration changes to site configuration history.

    Recording history on updates and deletes can be skipped by first setting
    the `skip_history_when_saving` attribute on the instace, e.g.:

      site_config.skip_history_when_saving = True
      site_config.save()

    Args:
        sender: sender of the signal i.e. SiteConfiguration model
        instance: SiteConfiguration instance associated with the current signal
        created (bool): True if a new record was created.
        **kwargs: extra key word arguments
    """
    # Skip writing history when asked by the caller.  This skip feature only
    # works for non-creates.
    if created or not hasattr(instance, "skip_history_when_saving"):
        SiteConfigurationHistory.objects.create(
            site=instance.site,
            site_values=instance.site_values,
            enabled=instance.enabled,
        )
<commit_msg>Use get_value instead deprecated values<commit_after>"""
Django models for site configurations.
"""

import collections
from logging import getLogger
import os

from django.conf import settings
from django.core.files.storage import get_storage_class
from django.contrib.sites.models import Site
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.core.files.storage import get_storage_class
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from jsonfield.fields import JSONField
from model_utils.models import TimeStampedModel

from .exceptions import TahoeConfigurationException

logger = getLogger(__name__)  # pylint: disable=invalid-name

def get_initial_sass_variables():
    """
    Proxy to `utils.get_initial_sass_variables` to avoid test-time Django errors.

    # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
    """
    from openedx.core.djangoapps.appsembler.sites import utils
    return utils.get_initial_sass_variables()

def get_initial_page_elements():
    """
    Proxy to `utils.get_initial_page_elements` to avoid test-time Django errors.

    # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
    """
    from openedx.core.djangoapps.appsembler.sites import utils
    return utils.get_initial_page_elements()

@python_2_unicode_compatible
class SiteConfiguration(models.Model):
    """
    Model for storing site configuration. These configuration override OpenEdx configurations and settings.
    e.g. You can override site name, logo image, favicon etc. using site configuration.

    Fields:
        site (OneToOneField): one to one field relating each configuration to a single site
        site_values (JSONField):  json field to store configurations for a site

    .. no_pii:
    """
    site = models.OneToOneField(Site, related_name='configuration', on_delete=models.CASCADE)
    enabled = models.BooleanField(default=False, verbose_name=u"Enabled")
    site_values = JSONField(
        null=False,
        blank=True,
        # The actual default value is determined by calling the given callable.
        # Therefore, the default here is just {}, since that is the result of
        # calling `dict`.
        default=dict,
        load_kwargs={'object_pairs_hook': collections.OrderedDict}
    )
    sass_variables = JSONField(blank=True, default=get_initial_sass_variables)
    page_elements = JSONField(blank=True, default=get_initial_page_elements)

    def __str__(self):
        return u"<SiteConfiguration: {site} >".format(site=self.site)  # xss-lint: disable=python-wrap-html

    def __repr__(self):
        return self.__str__()

    def save(self, **kwargs):
        # When creating a new object, save default microsite values. Not implemented as a default method on the field
        # because it depends on other fields that should be already filled.
        if not self.id:
            self.site_values = self.get_initial_microsite_values()

        # fix for a bug with some pages requiring uppercase platform_name variable
        self.site_values['PLATFORM_NAME'] = self.site_values.get('platform_name', '')

        # Set the default language code for new sites if missing
        # TODO: Move it to somewhere else like in AMC
        self.site_values['LANGUAGE_CODE'] = self.site_values.get('LANGUAGE_CODE', 'en')

        super(SiteConfiguration, self).save(**kwargs)

        # recompile SASS on every save
        self.compile_microsite_sass()
        return self

    def get_value(self, name, default=None):
        """
        Return Configuration value for the key specified as name argument.

        Function logs a message if configuration is not enabled or if there is an error retrieving a key.

        Args:
            name (str): Name of the key for which to return configuration value.
            default: default value tp return if key is not found in the configuration

        Returns:
            Configuration value for the given key or returns `None` if configuration is not enabled.
        """
        if self.enabled:
            try:
                return self.site_values.get(name, default)
            except AttributeError as error:
                logger.exception(u'Invalid JSON data. \n [%s]', error)
        else:
            logger.info(u"Site Configuration is not enabled for site (%s).", self.site)

        return default

    @classmethod
    def get_configuration_for_org(cls, org, select_related=None):
        """
        This returns a SiteConfiguration object which has an org_filter that matches
        the supplied org

        Args:
            org (str): Org to use to filter SiteConfigurations
            select_related (list or None): A list of values to pass as arguments to select_related
        """
        query = cls.objects.filter(site_values__contains=org, enabled=True).defer('page_elements', 'sass_variables').all()
        if select_related is not None:
            query = query.select_related(*select_related)
        for configuration in query:
            course_org_filter = configuration.get_value('course_org_filter', [])
            # The value of 'course_org_filter' can be configured as a string representing
            # a single organization or a list of strings representing multiple organizations.
            if not isinstance(course_org_filter, list):
                course_org_filter = [course_org_filter]
            if org in course_org_filter:
                return configuration
        return None

    @classmethod
    def get_value_for_org(cls, org, name, default=None):
        """
        This returns site configuration value which has an org_filter that matches
        what is passed in,

        Args:
            org (str): Course ord filter, this value will be used to filter out the correct site configuration.
            name (str): Name of the key for which to return configuration value.
            default: default value tp return if key is not found in the configuration

        Returns:
            Configuration value for the given key.
        """
        configuration = cls.get_configuration_for_org(org)
        if configuration is None:
            return default
        else:
            return configuration.get_value(name, default)

    @classmethod
    def get_all_orgs(cls):
        """
        This returns all of the orgs that are considered in site configurations, This can be used,
        for example, to do filtering.

        Returns:
            A set of all organizations present in site configuration.
        """
        org_filter_set = set()

        for configuration in cls.objects.filter(site_values__contains='course_org_filter', enabled=True).defer('page_elements', 'sass_variables').all():
            course_org_filter = configuration.get_value('course_org_filter', [])
            if not isinstance(course_org_filter, list):
                course_org_filter = [course_org_filter]
            org_filter_set.update(course_org_filter)
        return org_filter_set

    @classmethod
    def has_org(cls, org):
        """
        Check if the given organization is present in any of the site configuration.

        Returns:
            True if given organization is present in site configurations otherwise False.
        """
        return org in cls.get_all_orgs()

    def delete(self, using=None):
        self.delete_css_override()
        super(SiteConfiguration, self).delete(using=using)

    def compile_microsite_sass(self):
        # Importing `compile_sass` to avoid test-time Django errors.
        # TODO: Fix Site Configuration and Organizations hacks. https://github.com/appsembler/edx-platform/issues/329
        from openedx.core.djangoapps.appsembler.sites.utils import compile_sass
        css_output = compile_sass('main.scss', custom_branding=self._sass_var_override)
        file_name = self.get_value('css_overrides_file')

        if not file_name:
            if settings.TAHOE_SILENT_MISSING_CSS_CONFIG:
                return  # Silent the exception below on during testing
            else:
                raise TahoeConfigurationException(
                    'Missing `css_overrides_file` from SiteConfiguration for `{site}` config_id=`{id}`'.format(
                        site=self.site.domain,
                        id=self.id,
                    )
                )

        if settings.USE_S3_FOR_CUSTOMER_THEMES:
            storage_class = get_storage_class(settings.DEFAULT_FILE_STORAGE)
            storage = storage_class(
                location="customer_themes",
            )
            with storage.open(file_name, 'w') as f:
                f.write(css_output)
        else:
            theme_folder = os.path.join(settings.COMPREHENSIVE_THEME_DIRS[0], 'customer_themes')
            theme_file = os.path.join(theme_folder, file_name)
            with open(theme_file, 'w') as f:
                f.write(css_output)

    def get_css_url(self):
        if settings.USE_S3_FOR_CUSTOMER_THEMES:
            kwargs = {
                'location': "customer_themes",
            }
            storage = get_storage_class()(**kwargs)
            return storage.url(self.get_value('css_overrides_file'))
        else:
            return static("customer_themes/{}".format(self.get_value('css_overrides_file')))

    def set_sass_variables(self, entries):
        """
        Accepts a dict with the shape { var_name: value } and sets the SASS variables
        """
        for index, entry in enumerate(self.sass_variables):
            var_name = entry[0]
            if var_name in entries:
                new_value = (var_name, [entries[var_name], entries[var_name]])
                self.sass_variables[index] = new_value

    def delete_css_override(self):
        css_file = self.get_value('css_overrides_file')
        if css_file:
            try:
                if settings.USE_S3_FOR_CUSTOMER_THEMES:
                    kwargs = {
                        'location': "customer_themes",
                    }
                    storage = get_storage_class()(**kwargs)
                    storage.delete(self.get_value('css_overrides_file'))
                else:
                    os.remove(os.path.join(settings.COMPREHENSIVE_THEME_DIRS[0], css_file))
            except OSError:
                logger.warning("Can't delete CSS file {}".format(css_file))

    def _formatted_sass_variables(self):
        return " ".join(["{}: {};".format(var, val[0]) for var, val in self.sass_variables])

    def _sass_var_override(self, path):
        if 'branding-basics' in path:
            return [(path, self._formatted_sass_variables())]
        if 'customer-sass-input' in path:
            return [(path, self.get_value('customer_sass_input', ''))]
        return None

    def get_initial_microsite_values(self):
        domain_without_port_number = self.site.domain.split(':')[0]
        return {
            'platform_name': self.site.name,
            'css_overrides_file': "{}.css".format(domain_without_port_number),
            'ENABLE_COMBINED_LOGIN_REGISTRATION': True,
        }

def save_siteconfig_without_historical_record(siteconfig, *args, **kwargs):
    """
    Save model without saving a historical record

    Make sure you know what you're doing before you use this method.

    Note: this method is copied verbatim from django-simple-history.
    """
    siteconfig.skip_history_when_saving = True
    try:
        ret = siteconfig.save(*args, **kwargs)
    finally:
        del siteconfig.skip_history_when_saving
    return ret

@python_2_unicode_compatible
class SiteConfigurationHistory(TimeStampedModel):
    """
    This is an archive table for SiteConfiguration, so that we can maintain a history of
    changes. Note that the site field is not unique in this model, compared to SiteConfiguration.

    Fields:
        site (ForeignKey): foreign-key to django Site
        site_values (JSONField): json field to store configurations for a site

    .. no_pii:
    """
    site = models.ForeignKey(Site, related_name='configuration_histories', on_delete=models.CASCADE)
    enabled = models.BooleanField(default=False, verbose_name=u"Enabled")
    site_values = JSONField(
        null=False,
        blank=True,
        load_kwargs={'object_pairs_hook': collections.OrderedDict}
    )

    class Meta:
        get_latest_by = 'modified'
        ordering = ('-modified', '-created',)

    def __str__(self):
        # pylint: disable=line-too-long
        return u"<SiteConfigurationHistory: {site}, Last Modified: {modified} >".format(  # xss-lint: disable=python-wrap-html
            modified=self.modified,
            site=self.site,
        )

    def __repr__(self):
        return self.__str__()

@receiver(post_save, sender=SiteConfiguration)
def update_site_configuration_history(sender, instance, created, **kwargs):  # pylint: disable=unused-argument
    """
    Add site configuration changes to site configuration history.

    Recording history on updates and deletes can be skipped by first setting
    the `skip_history_when_saving` attribute on the instace, e.g.:

      site_config.skip_history_when_saving = True
      site_config.save()

    Args:
        sender: sender of the signal i.e. SiteConfiguration model
        instance: SiteConfiguration instance associated with the current signal
        created (bool): True if a new record was created.
        **kwargs: extra key word arguments
    """
    # Skip writing history when asked by the caller.  This skip feature only
    # works for non-creates.
    if created or not hasattr(instance, "skip_history_when_saving"):
        SiteConfigurationHistory.objects.create(
            site=instance.site,
            site_values=instance.site_values,
            enabled=instance.enabled,
        )
<|endoftext|>


<commit_before>import django
import pynvest_connect
from pynvest_investment import models

class Populate(object):
    def __init__(self, investment, fix_existing):
        self.investment = investment
        self.fix_existing = fix_existing

        self.dirty_dates = set()
        self.historicalprices = dict((historicalprice.date, historicalprice) for historicalprice in investment.historicalprice_set.all())
        self.dividends = self._historicalprice_subs(models.Dividend)
        self.splits = self._historicalprice_subs(models.Split)
        self.priceadjustments = self._historicalprice_subs(models.PriceAdjustment)

    @classmethod
    def all(cls, *args, **kwargs):
        populate = cls(*args, **kwargs)
        for func in [populate._prices, populate._dividends, populate._splits, populate._priceadjustment]:
            try:
                func()
            except pynvest_connect.CallNotSupportedException:
                pass
        return populate

    def _historicalprice_subs(self, Model):
        return dict((instance.historicalprice.date, instance)
                        for instance in Model.objects.select_related()
                                                     .filter(historicalprice__investment=self.investment))

    def _prices(self):
        for row in pynvest_connect.historical_prices(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            historicalprice = self.historicalprices.setdefault(row.date, models.HistoricalPrice(investment=self.investment, date=row.date))
            if (historicalprice.high  == row.high and
                historicalprice.low   == row.low and
                historicalprice.close == row.close):
                if self.fix_existing:
                    continue
                else:
                    break

            historicalprice.high = row.high
            historicalprice.low = row.low
            historicalprice.close = row.close
            historicalprice.save()
            self.dirty_dates.add(row.date)

    def _dividends(self):
        for row in pynvest_connect.dividends(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            dividend = self.dividends.setdefault(row.date, models.Dividend(historicalprice=self.historicalprices[row.date]))
            if dividend.amount == row.amount:
                if self.fix_existing:
                    continue
                else:
                    break

            dividend.amount = row.amount
            dividend.save()
            self.dirty_dates.add(row.date)

    def _splits(self):
        for row in pynvest_connect.splits(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            split = self.splits.setdefault(row.date, models.Split(historicalprice=self.historicalprices[row.date]))
            if split.before == row.before and split.after == row.after:
                if self.fix_existing:
                    continue
                else:
                    break

            split.before = row.before
            split.after = row.after
            split.save()
            self.dirty_dates.add(row.date)

    def _priceadjustment(self):
        if self.fix_existing or not self.priceadjustments:
            dates = sorted(self.historicalprices)
            last_raw = 1.0
        else:
            dates = sorted(set(self.historicalprices) - set(self.priceadjustments))
            last_raw = self.priceadjustments[max(self.priceadjustments)].raw

        for date in dates:
            if date in self.dividends:
                last_raw *= float(self.dividends[date].amount + self.historicalprices[date].close) / float(self.historicalprices[date].close)
            if date in self.splits:
                last_raw *= float(self.splits[date].after) / self.splits[date].before

            priceadjustment = self.priceadjustments.setdefault(date, models.PriceAdjustment(historicalprice=self.historicalprices[date]))
            if priceadjustment.raw == last_raw:
                continue

            priceadjustment.raw = last_raw
            priceadjustment.save()
            self.dirty_dates.add(date)

class Command(django.core.management.base.BaseCommand):
    help = 'Populates all prices for existing investments'

    def handle(self, *args, **options):
        fix_existing = options.pop('fix_existing', False)
        investments = models.Investment.objects.all()

        num_investments = len(investments)
        for (index, investment) in enumerate(investments):
            if isinstance(investment, basestring):
                investment, created = Investment.objects.get_or_create(symbol=investment, defaults={'name': 'Placeholder'})

            self.stdout.write('%3d/%d %-11s ' % (index + 1, num_investments, investment.symbol), ending='')

            with django.db.transaction.commit_on_success():
                populate = Populate.all(investment, fix_existing)

            if populate.dirty_dates:
                if len(populate.dirty_dates) > 1:
                    self.stdout.write(min(populate.dirty_dates).strftime('%Y-%m-%d - '), ending='')
                self.stdout.write(max(populate.dirty_dates).strftime('%Y-%m-%d'), ending='')

            self.stdout.write('')
<commit_msg>Removed half-baked 'fix_existing' populate<commit_after>import django
import pynvest_connect
from pynvest_investment import models

class Populate(object):
    def __init__(self, investment):
        self.investment = investment

        self.dirty_dates = set()
        self.historicalprices = dict((historicalprice.date, historicalprice) for historicalprice in investment.historicalprice_set.all())
        self.dividends = self._historicalprice_subs(models.Dividend)
        self.splits = self._historicalprice_subs(models.Split)
        self.priceadjustments = self._historicalprice_subs(models.PriceAdjustment)

    @classmethod
    def all(cls, *args, **kwargs):
        populate = cls(*args, **kwargs)
        for func in [populate._prices, populate._dividends, populate._splits, populate._priceadjustment]:
            try:
                func()
            except pynvest_connect.CallNotSupportedException:
                pass
        return populate

    def _historicalprice_subs(self, Model):
        return dict((instance.historicalprice.date, instance)
                        for instance in Model.objects.select_related()
                                                     .filter(historicalprice__investment=self.investment))

    def _prices(self):
        for row in pynvest_connect.historical_prices(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            historicalprice = self.historicalprices.setdefault(row.date, models.HistoricalPrice(investment=self.investment, date=row.date))
            if (historicalprice.high  == row.high and
                historicalprice.low   == row.low and
                historicalprice.close == row.close):
                break

            historicalprice.high = row.high
            historicalprice.low = row.low
            historicalprice.close = row.close
            historicalprice.save()
            self.dirty_dates.add(row.date)

    def _dividends(self):
        for row in pynvest_connect.dividends(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            dividend = self.dividends.setdefault(row.date, models.Dividend(historicalprice=self.historicalprices[row.date]))
            if dividend.amount == row.amount:
                break

            dividend.amount = row.amount
            dividend.save()
            self.dirty_dates.add(row.date)

    def _splits(self):
        for row in pynvest_connect.splits(self.investment.symbol, jurisdiction=self.investment.jurisdiction.symbol):
            split = self.splits.setdefault(row.date, models.Split(historicalprice=self.historicalprices[row.date]))
            if split.before == row.before and split.after == row.after:
                break

            split.before = row.before
            split.after = row.after
            split.save()
            self.dirty_dates.add(row.date)

    def _priceadjustment(self):
        if self.priceadjustments:
            dates = sorted(set(self.historicalprices) - set(self.priceadjustments))
            last_raw = self.priceadjustments[max(self.priceadjustments)].raw
        else:
            dates = sorted(self.historicalprices)
            last_raw = 1.0

        for date in dates:
            if date in self.dividends:
                last_raw *= float(self.dividends[date].amount + self.historicalprices[date].close) / float(self.historicalprices[date].close)
            if date in self.splits:
                last_raw *= float(self.splits[date].after) / self.splits[date].before

            priceadjustment = self.priceadjustments.setdefault(date, models.PriceAdjustment(historicalprice=self.historicalprices[date]))
            if priceadjustment.raw == last_raw:
                continue

            priceadjustment.raw = last_raw
            priceadjustment.save()
            self.dirty_dates.add(date)

class Command(django.core.management.base.BaseCommand):
    help = 'Populates all prices for existing investments'

    def handle(self, *args, **options):
        investments = models.Investment.objects.all()

        num_investments = len(investments)
        for (index, investment) in enumerate(investments):
            if isinstance(investment, basestring):
                investment, created = Investment.objects.get_or_create(symbol=investment, defaults={'name': 'Placeholder'})

            self.stdout.write('%3d/%d %-11s ' % (index + 1, num_investments, investment.symbol), ending='')

            with django.db.transaction.commit_on_success():
                populate = Populate.all(investment)

            if populate.dirty_dates:
                if len(populate.dirty_dates) > 1:
                    self.stdout.write(min(populate.dirty_dates).strftime('%Y-%m-%d - '), ending='')
                self.stdout.write(max(populate.dirty_dates).strftime('%Y-%m-%d'), ending='')

            self.stdout.write('')
<|endoftext|>


<commit_before>"""
English Word Segmentation in Python

Word segmentation is the process of dividing a phrase without spaces back
into its constituent parts. For example, consider a phrase like "thisisatest".
For humans, it's relatively easy to parse. This module makes it easy for
machines too. Use `segment` to parse a phrase into its parts:

>>> from wordsegment import segment
>>> segment('thisisatest')
['this', 'is', 'a', 'test']
"""
import sys
from math import log10
from functools import wraps
import re
from collections import Counter
import operator
import codecs
import pickle
from metaphone import doublemetaphone

if sys.hexversion < 0x03000000:
    range = xrange

unigram_counts,bigram_counts, total_count = ("",) * 3

def read_text_for_segmentation_file(input_file):
    return codecs.open(input_file, 'r').read().lower()

def dump_into_pickle_file(output_file, word_counts):
    with open(output_file, 'wb') as handle:
        pickle.dump(dict(word_counts), handle)

def word_count(text,dd min_size, max_size, flag):
    return Counter([word_or_metaphone(word, flag) for word in re.findall(r'\w+', text) if (len(word) < (abs(max_size) + 1) and len(word) > (abs(min_size) - 1) and not unicode(word, 'utf-8').isnumeric())])

def extract_segmentation_file_from_text(input_file, output_file, min_size, max_size, **kwargs):
    text          = read_text_for_segmentation_file(input_file)
    word_counts   = word_count(text, min_size, max_size, True) if kwargs.get('metaphone') else word_count(text, min_size, max_size, False)
    dump_into_pickle_file(output_file, word_counts)

def word_or_metaphone(word, flag):
    return get_metaphone_from_word(word) if flag else word

def load_data_from_pickle_file(filename):
    with open(filename, 'rb') as handle:
        words = pickle.load(handle)
    return words

def get_metaphone_from_word(word):
  return doublemetaphone(word)[0] if len(doublemetaphone(word)[0]) > 1 else doublemetaphone(word)[1]

def load_data_from_text_file(filename):
    """Read `filename` and parse tab-separated file of (word, count) pairs."""
    with open(filename) as fptr:
        lines = (line.split('\t') for line in fptr)
        return dict((word, number) for word, number in lines)

def change_data_values_to_float(data, factor):
    return {key: float(value * factor) for key, value in data.items()}

def memoize(func):
    """Memoize arguments to function `func`."""
    cache = dict()
    @wraps(func)
    def wrapper(*args):
        if args not in cache:
            cache[args] = func(*args)
        return cache[args]
    return wrapper

def divide(text, limit=50):
    for pos in range(1, min(len(text), limit) + 1):
        yield (text[:pos], text[pos:])

def score(word, prev=None):
    if prev is None:
        if word in unigram_counts:
            return unigram_counts[word] / total_count
        else:
            return 10.0 / (total_count * 10 ** len(word))
    else:
        bigram = '{0} {1}'.format(prev, word)
        if bigram in bigram_counts and prev in unigram_counts:
            return bigram_counts[bigram] / total_count / score(prev)
        else:
            return score(word)

def set_segmentation_data(segment_data, **kwargs):
    global unigram_counts, total_count, bigram_counts
    factor                        = kwargs.get('factor') if kwargs.get('factor') > 1 else 1
    unigram_counts, bigram_counts = (change_data_values_to_float(segment_data, factor),) * 2
    total_count                   = sum(unigram_counts.values())

def segment(text, **kwargs):
    factor                        = kwargs.get('factor') if kwargs.get('factor') > 1 else 1
    if bool(unigram_counts) is False or 'segment_data' in kwargs:
        set_segmentation_data.kwargs.get('segment_data', factor = factor)

    @memoize
    def search(text, prev='<S>'):
        if text == '':
            return 0.0, []

        def candidates():
            for prefix, suffix in divide(text):
                prefix_score = log10(score(prefix, prev))
                suffix_score, suffix_words = search(suffix, prefix)
                yield (prefix_score + suffix_score, [prefix] + suffix_words)

        return max(candidates())

    result_score, result_words = search(text)

    return result_words<commit_msg>user name check<commit_after>"""
English Word Segmentation in Python

Word segmentation is the process of dividing a phrase without spaces back
into its constituent parts. For example, consider a phrase like "thisisatest".
For humans, it's relatively easy to parse. This module makes it easy for
machines too. Use `segment` to parse a phrase into its parts:

>>> from wordsegment import segment
>>> segment('thisisatest')
['this', 'is', 'a', 'test']
"""
import sys
from math import log10
from functools import wraps
import re
from collections import Counter
import operator
import codecs
import pickle
from metaphone import doublemetaphone

if sys.hexversion < 0x03000000:
    range = xrange

unigram_counts,bigram_counts, total_count = ("",) * 3

def read_text_for_segmentation_file(input_file):
    return codecs.open(input_file, 'r').read().lower()

def dump_into_pickle_file(output_file, word_counts):
    with open(output_file, 'wb') as handle:
        pickle.dump(dict(word_counts), handle)

def word_count(text, min_size, max_size, flag):
    return Counter([word_or_metaphone(word, flag) for word in re.findall(r'\w+', text) if (len(word) < (abs(max_size) + 1) and len(word) > (abs(min_size) - 1) and not unicode(word, 'utf-8').isnumeric())])

def extract_segmentation_file_from_text(input_file, output_file, min_size, max_size, **kwargs):
    text          = read_text_for_segmentation_file(input_file)
    word_counts   = word_count(text, min_size, max_size, True) if kwargs.get('metaphone') else word_count(text, min_size, max_size, False)
    dump_into_pickle_file(output_file, word_counts)

def word_or_metaphone(word, flag):
    return get_metaphone_from_word(word) if flag else word

def load_data_from_pickle_file(filename):
    with open(filename, 'rb') as handle:
        words = pickle.load(handle)
    return words

def get_metaphone_from_word(word):
  return doublemetaphone(word)[0] if len(doublemetaphone(word)[0]) > 1 else doublemetaphone(word)[1]

def load_data_from_text_file(filename):
    """Read `filename` and parse tab-separated file of (word, count) pairs."""
    with open(filename) as fptr:
        lines = (line.split('\t') for line in fptr)
        return dict((word, number) for word, number in lines)

def change_data_values_to_float(data, factor):
    return {key: float(value * factor) for key, value in data.items()}

def memoize(func):
    """Memoize arguments to function `func`."""
    cache = dict()
    @wraps(func)
    def wrapper(*args):
        if args not in cache:
            cache[args] = func(*args)
        return cache[args]
    return wrapper

def divide(text, limit=50):
    for pos in range(1, min(len(text), limit) + 1):
        yield (text[:pos], text[pos:])

def score(word, prev=None):
    if prev is None:
        if word in unigram_counts:
            return unigram_counts[word] / total_count
        else:
            return 10.0 / (total_count * 10 ** len(word))
    else:
        bigram = '{0} {1}'.format(prev, word)
        if bigram in bigram_counts and prev in unigram_counts:
            return bigram_counts[bigram] / total_count / score(prev)
        else:
            return score(word)

def set_segmentation_data(segment_data, **kwargs):
    global unigram_counts, total_count, bigram_counts
    factor                        = kwargs.get('factor') if kwargs.get('factor') > 1 else 1
    unigram_counts, bigram_counts = (change_data_values_to_float(segment_data, factor),) * 2
    total_count                   = sum(unigram_counts.values())

def segment(text, **kwargs):
    factor                        = kwargs.get('factor') if kwargs.get('factor') > 1 else 1
    if bool(unigram_counts) is False or 'segment_data' in kwargs:
        set_segmentation_data.kwargs.get('segment_data', factor = factor)

    @memoize
    def search(text, prev='<S>'):
        if text == '':
            return 0.0, []

        def candidates():
            for prefix, suffix in divide(text):
                prefix_score = log10(score(prefix, prev))
                suffix_score, suffix_words = search(suffix, prefix)
                yield (prefix_score + suffix_score, [prefix] + suffix_words)

        return max(candidates())

    result_score, result_words = search(text)

    return result_words<|endoftext|>