<commit_before>from datetime import datetime, date
from decimal import Decimal
import json
from django.db.models.query import QuerySet
from django.db.models import Model
from django.db.models.fields.related import ForeignKey
from django.db.models.fields.files import FieldFile

from jsonate.exceptions import CouldntSerialize

# Custom encoder using a list of mapping functions
type_map = []
class JsonateEncoder(json.JSONEncoder):
    def default(self, obj):
        for obj_type, mapper in type_map:
            if isinstance(obj, obj_type):
                # recurse until we get to an object the encoder
                # can handle natively
                try:
                    return self.default(mapper(obj))
                except CouldntSerialize:
                    pass
        try:
            # this will handle the non-string things that
            # the default encoder can handle (like numbers, booleans etc)
            return super(JsonateEncoder, self).default(obj)
        except TypeError:
            return obj

# Decorator for registering mapping functions
def register_typemap(obj_type):
    def wrapper(fn):
        type_map.append((obj_type, fn))
        return fn
    return wrapper

# helper function for getting the correct fields to serialize
def jsonate_fields(model):
    all_fields = tuple(f.name for f in model._meta.fields)

    excluded = getattr(model._meta, "jsonate_exclude", ())
    fields = getattr(model._meta, 'jsonate_fields', all_fields)

    serialize = set(fields).difference(set(excluded))

    return tuple(field for field in model._meta.fields
                    if field.name in serialize)

#########################
##  Mapping functions  ##
#########################

# If the object knows how to serialize itself... let
# it do it's thing
@register_typemap(object)
def map_object(obj):
    to_json = getattr(obj, "to_json", getattr(obj, "toJSON", None))
    if to_json is None:
        raise CouldntSerialize
    return to_json()

@register_typemap(QuerySet)
def map_queryset(obj):
    # if the model wants to serialize itself, go with that...
    if hasattr(obj.model, 'to_json') or hasattr(obj.model, 'toJSON'):
        return list(obj)

    #
    # otherwise using values is faster
    #

    if not obj.exists():
        return []

    if isinstance(obj[0], dict):
        fields = obj[0].keys()
    else:
        fields = [field.name for field in jsonate_fields(obj.model)]

    return list(
        obj.values(*fields)
    )

@register_typemap(Model)
def map_model_instance(obj):
    d = {}
    for field in jsonate_fields(obj):
        value = getattr(obj, field.name)
        if isinstance(field, ForeignKey) and value is not None:
            value = value._get_pk_val()
        d[field.name] = value
    return d

@register_typemap(set)
def map_set(obj):
    return list(obj)

@register_typemap(FieldFile)
def map_filefield_file(obj):
    return obj.name

@register_typemap(date)
@register_typemap(datetime)
def map_datetime(obj):
    return obj.isoformat()

@register_typemap(Decimal)
def map_decimal(obj):
    return float(obj)<commit_msg>faster way to check type + mind ver. < 1.9<commit_after>from datetime import datetime, date
from decimal import Decimal
import json

from django.db.models.query import QuerySet

import django
django_19 = (django.VERSION >= (1,9))

if django_19:
    from django.db.models.query import ModelIterable
else:
    from django.db.models.query import ValuesQuerySet

from django.db.models import Model
from django.db.models.fields.related import ForeignKey
from django.db.models.fields.files import FieldFile

from jsonate.exceptions import CouldntSerialize

# Custom encoder using a list of mapping functions
type_map = []
class JsonateEncoder(json.JSONEncoder):
    def default(self, obj):
        for obj_type, mapper in type_map:
            if isinstance(obj, obj_type):
                # recurse until we get to an object the encoder
                # can handle natively
                try:
                    return self.default(mapper(obj))
                except CouldntSerialize:
                    pass
        try:
            # this will handle the non-string things that
            # the default encoder can handle (like numbers, booleans etc)
            return super(JsonateEncoder, self).default(obj)
        except TypeError:
            return obj

# Decorator for registering mapping functions
def register_typemap(obj_type):
    def wrapper(fn):
        type_map.append((obj_type, fn))
        return fn
    return wrapper

# helper function for getting the correct fields to serialize
def jsonate_fields(model):
    all_fields = tuple(f.name for f in model._meta.fields)

    excluded = getattr(model._meta, "jsonate_exclude", ())
    fields = getattr(model._meta, 'jsonate_fields', all_fields)

    serialize = set(fields).difference(set(excluded))

    return tuple(field for field in model._meta.fields
                    if field.name in serialize)

#########################
##  Mapping functions  ##
#########################

# If the object knows how to serialize itself... let
# it do it's thing
@register_typemap(object)
def map_object(obj):
    to_json = getattr(obj, "to_json", getattr(obj, "toJSON", None))
    if to_json is None:
        raise CouldntSerialize
    return to_json()

if not django_19:
    # Must come before map_queryset because ValuesQuerySet is
    # a subclass of Queryset and will cause an infinite loop :(
    @register_typemap(ValuesQuerySet)
    def map_values_queryset(obj):
        return list(obj)

@register_typemap(QuerySet)
def map_queryset(obj):
    # if the model wants to serialize itself, go with that...
    if hasattr(obj.model, 'to_json') or hasattr(obj.model, 'toJSON'):
        return list(obj)

    # otherwise using values is faster
    if django_19:
        if obj._iterable_class == ModelIterable:

            fields = jsonate_fields(obj.model)
            obj = obj.values(*[field.name for field in fields])

        return list(obj)

    else:
        fields = jsonate_fields(obj.model)
        return obj.values(*[field.name for field in fields])

@register_typemap(Model)
def map_model_instance(obj):
    d = {}
    for field in jsonate_fields(obj):
        value = getattr(obj, field.name)
        if isinstance(field, ForeignKey) and value is not None:
            value = value._get_pk_val()
        d[field.name] = value
    return d

@register_typemap(set)
def map_set(obj):
    return list(obj)

@register_typemap(FieldFile)
def map_filefield_file(obj):
    return obj.name

@register_typemap(date)
@register_typemap(datetime)
def map_datetime(obj):
    return obj.isoformat()

@register_typemap(Decimal)
def map_decimal(obj):
    return float(obj)<|endoftext|>


<commit_before># coding: utf-8
"""A tornado based Jupyter lab server."""

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import json
import os
from tornado import web

from notebook.base.handlers import IPythonHandler, FileFindHandler
from jinja2 import FileSystemLoader
from notebook.utils import url_path_join as ujoin
from .commands import _get_build_dir, _get_config, _get_config_dir

#-----------------------------------------------------------------------------
# Module globals
#-----------------------------------------------------------------------------

DEV_NOTE_NPM = """It looks like you're running JupyterLab from source.
If you're working on the TypeScript sources of JupyterLab, try running

    npm run watch

from the JupyterLab repo directory in another terminal window to have the
system incrementally watch and build JupyterLab's TypeScript for you, as you
make changes.
"""

HERE = os.path.dirname(__file__)
FILE_LOADER = FileSystemLoader(HERE)
PREFIX = '/lab'

class LabHandler(IPythonHandler):
    """Render the JupyterLab View."""

    def initialize(self, page_config_data, built_files):
        self.page_config_data = page_config_data
        self.built_files = built_files

    @web.authenticated
    def get(self):
        config = self._get_lab_config()
        self.write(self.render_template('lab.html', **config))

    def _get_lab_config(self):
        """Get the config data for the page template."""
        static_prefix = ujoin(self.base_url, PREFIX)

        bundles = [ujoin(static_prefix, name + '.bundle.js') for name in
                   ['main']]

        # Only load CSS files if they exist.
        css_files = []
        for css_file in ['main.css']:
            if os.path.isfile(os.path.join(self.built_files, css_file)):
                css_files.append(ujoin(static_prefix, css_file))

        configData = dict(self.page_config_data)
        configData.setdefault('ignorePlugins', [])
        configData.update(dict(
            terminalsAvailable=self.settings.get('terminals_available', False),
        ))

        mathjax_config = self.settings.get('mathjax_config',
                                           'TeX-AMS_HTML-full,Safe')
        config = dict(
            static_prefix=static_prefix,
            page_title='JupyterLab Alpha Preview',
            mathjax_url=self.mathjax_url,
            mathjax_config=mathjax_config,
            jupyterlab_css=css_files,
            jupyterlab_bundles=bundles
        )
        config['jupyterlab_config'] = configData
        return config

    def get_template(self, name):
        return FILE_LOADER.load(self.settings['jinja2_env'], name)

def add_handlers(app):
    """Add the appropriate handlers to the web app.
    """
    web_app = app.web_app
    base_url = web_app.settings['base_url']
    prefix = ujoin(base_url, PREFIX)

    # Handle page config data.
    config = _get_config()
    page_config_data = web_app.settings.get('page_config_data', {})
    page_config_file = os.path.join(_get_config_dir(), 'page_config_data.json')
    if os.path.exists(page_config_file):
        with open(page_config_file) as fid:
            page_config_data.update(json.load(fid))

    built_files = _get_build_dir(config)

    # Check for dev mode.
    dev_mode = False
    if hasattr(app, 'dev_mode'):
        dev_mode = app.dev_mode

    if not os.path.exists(built_files) or dev_mode:
        built_files = os.path.join(HERE, 'build')

    handlers = [
        (prefix + r'/?', LabHandler, {
            'page_config_data': page_config_data,
            'built_files': built_files
        }),
        (prefix + r"/(.*)", FileFindHandler, {
            'path': built_files
        })
    ]
    web_app.add_handlers(".*$", handlers)

def load_jupyter_server_extension(nbapp):
    """Load the JupyterLab server extension.
    """
    # Print messages.
    nbapp.log.info('JupyterLab alpha preview extension loaded from %s' % HERE)
    base_dir = os.path.realpath(os.path.join(HERE, '..'))
    dev_mode = os.path.exists(os.path.join(base_dir, '.git'))
    if dev_mode:
        nbapp.log.info(DEV_NOTE_NPM)

    add_handlers(nbapp)
<commit_msg>wip use jupyterlab-launcher<commit_after># coding: utf-8
"""A tornado based Jupyter lab server."""

# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import json
import os

from jinja2 import FileSystemLoader
from jupyterlab_launcher import add_handlers

from .commands import _get_build_dir, _get_config, DEFAULT_CONFIG_PATH

#-----------------------------------------------------------------------------
# Module globals
#-----------------------------------------------------------------------------

DEV_NOTE_NPM = """It looks like you're running JupyterLab from source.
If you're working on the TypeScript sources of JupyterLab, try running

    npm run watch

from the JupyterLab repo directory in another terminal window to have the
system incrementally watch and build JupyterLab's TypeScript for you, as you
make changes.
"""

HERE = os.path.dirname(__file__)
FILE_LOADER = FileSystemLoader(HERE)
PREFIX = '/lab'

def load_jupyter_server_extension(nbapp):
    """Load the JupyterLab server extension.
    """
    # Print messages.
    nbapp.log.info('JupyterLab alpha preview extension loaded from %s' % HERE)
    base_dir = os.path.realpath(os.path.join(HERE, '..'))
    dev_mode = os.path.exists(os.path.join(base_dir, '.git'))
    if dev_mode:
        nbapp.log.info(DEV_NOTE_NPM)

    web_app = nbapp.web_app

    # Handle page config data.
    config_dir = getattr(nbapp, 'lab_config_dir', DEFAULT_CONFIG_PATH)
    if 'LabApp' in nbapp.config:
        if 'lab_config_dir' in nbapp.config['LabApp']:
            config_dir = nbapp.config['LabApp']['lab_config_dir']

    build_config = _get_config(config_dir)
    page_config_file = os.path.join(config_dir, 'page_config_data.json')
    build_dir = _get_build_dir(build_config)

    # Check for dev mode.
    dev_mode = False
    if hasattr(nbapp, 'dev_mode'):
        dev_mode = nbapp.dev_mode

    if not os.path.exists(build_dir) or dev_mode:
        print('Serving local JupyterLab files')
        build_dir = os.path.join(HERE, 'build')

    add_handlers(
        web_app, page_config_file, build_dir, 'JupyterLab Alpha Preview',
        PREFIX
    )
<|endoftext|>


<commit_before>__all__ = [
    'opinions',
    'oral_args',
]

__version__ = "1.1.9.5"

__title__ = "juriscraper"
__description__ = "An API to scrape American court websites for metadata."
__uri__ = "https://github.com/freelawproject/juriscraper"

__author__ = "Mike Lissner"
__email__ = "info@free.law"

__license__ = "BSD"
__copyright__ = "Copyright (c) Free Law Project"
<commit_msg>Version bump.<commit_after>__all__ = [
    'opinions',
    'oral_args',
]

__version__ = "1.1.9.6"

__title__ = "juriscraper"
__description__ = "An API to scrape American court websites for metadata."
__uri__ = "https://github.com/freelawproject/juriscraper"

__author__ = "Mike Lissner"
__email__ = "info@free.law"

__license__ = "BSD"
__copyright__ = "Copyright (c) Free Law Project"
<|endoftext|>


<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import webnotes
from webnotes import _
from webnotes.utils import getdate, nowdate, flt

class AccountsReceivableReport(object):
	def __init__(self, filters=None):
		self.filters = webnotes._dict(filters or {})
		self.filters.report_date = getdate(self.filters.report_date or nowdate())
		self.age_as_on = getdate(nowdate()) \
			if self.filters.report_date > getdate(nowdate()) \
			else self.filters.report_date

	def run(self):
		return self.get_columns(), self.get_data()

	def get_columns(self):
		columns = [
			"Posting Date:Date:80", "Account:Link/Account:150",
			"Voucher Type::110", "Voucher No::120", "::30",
			"Due Date:Date:80",
			"Invoiced Amount:Currency:100", "Payment Received:Currency:100",
			"Outstanding Amount:Currency:100", "Age:Int:50", "0-30:Currency:100",
			"30-60:Currency:100", "60-90:Currency:100", "90-Above:Currency:100"
		]

		naming_by = self.get_selling_settings()
		if naming_by[0].value == "Naming Series":
			columns += ["Customer:Link/Customer:200"]

		columns += ["Customer Name::110", "Territory:Link/Territory:80", "Remarks::200"]

		return columns

	def get_data(self):
		data = []
		future_vouchers = self.get_entries_after(self.filters.report_date)
		for gle in self.get_entries_till(self.filters.report_date):
			if self.is_receivable(gle, future_vouchers):
				outstanding_amount = self.get_outstanding_amount(gle, self.filters.report_date)
				if abs(outstanding_amount) > 0.01:
					due_date = self.get_due_date(gle)
					invoiced_amount = gle.debit if (gle.debit > 0) else 0
					payment_received = invoiced_amount - outstanding_amount
					row = [gle.posting_date, gle.account,
						gle.voucher_type, gle.voucher_no, due_date,
						invoiced_amount, payment_received,
						outstanding_amount]
					entry_date = due_date if self.filters.ageing_based_on == "Due Date" \
						else gle.posting_date
					row += get_ageing_data(self.age_as_on, entry_date, outstanding_amount)

					naming_by = self.get_selling_settings()
					if naming_by[0].value == "Naming Series":
						row += [self.get_customer(gle.account)]

					row += [self.get_customer_name(gle.account), self.get_territory(gle.account), gle.remarks]
					data.append(row)

		for i in range(0,len(data)):
			data[i].insert(4, """""" \
				% ("/".join(["#Form", data[i][2], data[i][3]]),))

		return data

	def get_selling_settings(self):
		return webnotes.conn.sql("""select value from `tabSingles` where
			doctype='Selling Settings' and field='cust_master_name'""", as_dict=1)

	def get_entries_after(self, report_date):
		# returns a distinct list
		return list(set([(e.voucher_type, e.voucher_no) for e in self.get_gl_entries()
			if getdate(e.posting_date) > report_date]))

	def get_entries_till(self, report_date):
		# returns a generator
		return (e for e in self.get_gl_entries()
			if getdate(e.posting_date) <= report_date)

	def is_receivable(self, gle, future_vouchers):
		return ((not gle.against_voucher) or (gle.against_voucher==gle.voucher_no) or
			((gle.against_voucher_type, gle.against_voucher) in future_vouchers))

	def get_outstanding_amount(self, gle, report_date):
		payment_received = 0.0
		for e in self.get_gl_entries_for(gle.account, gle.voucher_type, gle.voucher_no):
			if getdate(e.posting_date) <= report_date and e.name!=gle.name:
				payment_received += (flt(e.credit) - flt(e.debit))

		return flt(gle.debit) - flt(gle.credit) - payment_received

	def get_customer(self, account):
		return self.get_account_map().get(account).get("customer") or ""

	def get_customer_name(self, account):
		return self.get_account_map().get(account).get("customer_name") or ""

	def get_territory(self, account):
		return self.get_account_map().get(account).get("territory") or ""

	def get_account_map(self):
		if not hasattr(self, "account_map"):
			self.account_map = dict(((r.name, r) for r in webnotes.conn.sql("""select
				acc.name, cust.name as customer, cust.customer_name, cust.territory
				from `tabAccount` acc, `tabCustomer` cust
				where acc.master_type="Customer"
				and cust.name=acc.master_name""", as_dict=True)))

		return self.account_map

	def get_due_date(self, gle):
		if not hasattr(self, "invoice_due_date_map"):
			# TODO can be restricted to posting date
			self.invoice_due_date_map = dict(webnotes.conn.sql("""select name, due_date
				from `tabSales Invoice` where docstatus=1"""))

		return gle.voucher_type == "Sales Invoice" \
			and self.invoice_due_date_map.get(gle.voucher_no) or ""

	def get_gl_entries(self):
		if not hasattr(self, "gl_entries"):
			conditions, values = self.prepare_conditions()
			self.gl_entries = webnotes.conn.sql("""select * from `tabGL Entry`
				where docstatus < 2 {} order by posting_date, account""".format(conditions),
				values, as_dict=True)

		return self.gl_entries

	def prepare_conditions(self):
		conditions = [""]
		values = {}

		if self.filters.company:
			conditions.append("company=%(company)s")
			values["company"] = self.filters.company

		if self.filters.account:
			conditions.append("account=%(account)s")
			values["account"] = self.filters.account
		else:
			account_map = self.get_account_map()
			if not account_map:
				webnotes.throw(_("No Customer Accounts found."))
			else:
				accounts_list = ['"{}"'.format(ac) for ac in account_map]
				conditions.append("account in ({})".format(", ".join(accounts_list)))

		return " and ".join(conditions), values

	def get_gl_entries_for(self, account, against_voucher_type, against_voucher):
		if not hasattr(self, "gl_entries_map"):
			self.gl_entries_map = {}
			for gle in self.get_gl_entries():
				if gle.against_voucher_type and gle.against_voucher:
					self.gl_entries_map.setdefault(gle.account, {})\
						.setdefault(gle.against_voucher_type, {})\
						.setdefault(gle.against_voucher, [])\
						.append(gle)

		return self.gl_entries_map.get(account, {})\
			.get(against_voucher_type, {})\
			.get(against_voucher, [])

def execute(filters=None):
	return AccountsReceivableReport(filters).run()

def get_ageing_data(age_as_on, entry_date, outstanding_amount):
	# [0-30, 30-60, 60-90, 90-above]
	outstanding_range = [0.0, 0.0, 0.0, 0.0]
	if not (age_as_on and entry_date):
		return [0] + outstanding_range

	age = (getdate(age_as_on) - getdate(entry_date)).days or 0
	index = None
	for i, days in enumerate([30, 60, 90]):
		if age <= days:
			index = i
			break

	if index is None: index = 3
	outstanding_range[index] = outstanding_amount

	return [age] + outstanding_range<commit_msg>fixed accounts receivable<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import webnotes
from webnotes import _
from webnotes.utils import getdate, nowdate, flt

class AccountsReceivableReport(object):
	def __init__(self, filters=None):
		self.filters = webnotes._dict(filters or {})
		self.filters.report_date = getdate(self.filters.report_date or nowdate())
		self.age_as_on = getdate(nowdate()) \
			if self.filters.report_date > getdate(nowdate()) \
			else self.filters.report_date

	def run(self):
		return self.get_columns(), self.get_data()

	def get_columns(self):
		columns = [
			"Posting Date:Date:80", "Account:Link/Account:150",
			"Voucher Type::110", "Voucher No::120", "::30",
			"Due Date:Date:80",
			"Invoiced Amount:Currency:100", "Payment Received:Currency:100",
			"Outstanding Amount:Currency:100", "Age:Int:50", "0-30:Currency:100",
			"30-60:Currency:100", "60-90:Currency:100", "90-Above:Currency:100"
		]

		if self.get_customer_naming() == "Naming Series":
			columns += ["Customer:Link/Customer:200"]

		columns += ["Customer Name::110", "Territory:Link/Territory:80", "Remarks::200"]

		return columns

	def get_data(self):
		data = []
		future_vouchers = self.get_entries_after(self.filters.report_date)
		for gle in self.get_entries_till(self.filters.report_date):
			if self.is_receivable(gle, future_vouchers):
				outstanding_amount = self.get_outstanding_amount(gle, self.filters.report_date)
				if abs(outstanding_amount) > 0.01:
					due_date = self.get_due_date(gle)
					invoiced_amount = gle.debit if (gle.debit > 0) else 0
					payment_received = invoiced_amount - outstanding_amount
					row = [gle.posting_date, gle.account,
						gle.voucher_type, gle.voucher_no, due_date,
						invoiced_amount, payment_received,
						outstanding_amount]
					entry_date = due_date if self.filters.ageing_based_on == "Due Date" \
						else gle.posting_date
					row += get_ageing_data(self.age_as_on, entry_date, outstanding_amount)

					if self.get_customer_naming() == "Naming Series":
						row += [self.get_customer(gle.account)]

					row += [self.get_customer_name(gle.account), self.get_territory(gle.account), gle.remarks]
					data.append(row)

		for i in range(0,len(data)):
			data[i].insert(4, """""" \
				% ("/".join(["#Form", data[i][2], data[i][3]]),))

		return data

	def get_customer_naming(self):
		return webnotes.conn.get_value("Selling Settings", None, "cust_master_name")

	def get_entries_after(self, report_date):
		# returns a distinct list
		return list(set([(e.voucher_type, e.voucher_no) for e in self.get_gl_entries()
			if getdate(e.posting_date) > report_date]))

	def get_entries_till(self, report_date):
		# returns a generator
		return (e for e in self.get_gl_entries()
			if getdate(e.posting_date) <= report_date)

	def is_receivable(self, gle, future_vouchers):
		return ((not gle.against_voucher) or (gle.against_voucher==gle.voucher_no) or
			((gle.against_voucher_type, gle.against_voucher) in future_vouchers))

	def get_outstanding_amount(self, gle, report_date):
		payment_received = 0.0
		for e in self.get_gl_entries_for(gle.account, gle.voucher_type, gle.voucher_no):
			if getdate(e.posting_date) <= report_date and e.name!=gle.name:
				payment_received += (flt(e.credit) - flt(e.debit))

		return flt(gle.debit) - flt(gle.credit) - payment_received

	def get_customer(self, account):
		return self.get_account_map().get(account).get("customer") or ""

	def get_customer_name(self, account):
		return self.get_account_map().get(account).get("customer_name") or ""

	def get_territory(self, account):
		return self.get_account_map().get(account).get("territory") or ""

	def get_account_map(self):
		if not hasattr(self, "account_map"):
			self.account_map = dict(((r.name, r) for r in webnotes.conn.sql("""select
				acc.name, cust.name as customer, cust.customer_name, cust.territory
				from `tabAccount` acc, `tabCustomer` cust
				where acc.master_type="Customer"
				and cust.name=acc.master_name""", as_dict=True)))

		return self.account_map

	def get_due_date(self, gle):
		if not hasattr(self, "invoice_due_date_map"):
			# TODO can be restricted to posting date
			self.invoice_due_date_map = dict(webnotes.conn.sql("""select name, due_date
				from `tabSales Invoice` where docstatus=1"""))

		return gle.voucher_type == "Sales Invoice" \
			and self.invoice_due_date_map.get(gle.voucher_no) or ""

	def get_gl_entries(self):
		if not hasattr(self, "gl_entries"):
			conditions, values = self.prepare_conditions()
			self.gl_entries = webnotes.conn.sql("""select * from `tabGL Entry`
				where docstatus < 2 {} order by posting_date, account""".format(conditions),
				values, as_dict=True)

		return self.gl_entries

	def prepare_conditions(self):
		conditions = [""]
		values = {}

		if self.filters.company:
			conditions.append("company=%(company)s")
			values["company"] = self.filters.company

		if self.filters.account:
			conditions.append("account=%(account)s")
			values["account"] = self.filters.account
		else:
			account_map = self.get_account_map()
			if not account_map:
				webnotes.throw(_("No Customer Accounts found."))
			else:
				accounts_list = ['"{}"'.format(ac) for ac in account_map]
				conditions.append("account in ({})".format(", ".join(accounts_list)))

		return " and ".join(conditions), values

	def get_gl_entries_for(self, account, against_voucher_type, against_voucher):
		if not hasattr(self, "gl_entries_map"):
			self.gl_entries_map = {}
			for gle in self.get_gl_entries():
				if gle.against_voucher_type and gle.against_voucher:
					self.gl_entries_map.setdefault(gle.account, {})\
						.setdefault(gle.against_voucher_type, {})\
						.setdefault(gle.against_voucher, [])\
						.append(gle)

		return self.gl_entries_map.get(account, {})\
			.get(against_voucher_type, {})\
			.get(against_voucher, [])

def execute(filters=None):
	return AccountsReceivableReport(filters).run()

def get_ageing_data(age_as_on, entry_date, outstanding_amount):
	# [0-30, 30-60, 60-90, 90-above]
	outstanding_range = [0.0, 0.0, 0.0, 0.0]
	if not (age_as_on and entry_date):
		return [0] + outstanding_range

	age = (getdate(age_as_on) - getdate(entry_date)).days or 0
	index = None
	for i, days in enumerate([30, 60, 90]):
		if age <= days:
			index = i
			break

	if index is None: index = 3
	outstanding_range[index] = outstanding_amount

	return [age] + outstanding_range<|endoftext|>


<commit_before>import copy, time
from threading import RLock
from . import ac_server_protocol
from .ac_server_helpers import DictToClass

class ACUdpMonitor:
    def __init__(self):
        # pluginId = 0 -> this plugin
        # pluginId = 1 -> proxied plugin
        self.HistoryInfo = DictToClass
        self.InfoRequest = DictToClass
        self.lock = RLock()
        self.reset()

    def reset(self, carId = None):
        with self.lock:
            if carId is None:
                self.intervals = [0,0]
                self.cu_history = [{},{}] # [pluginId][carId]
                self.info_requests = []
            else:
                if carId in self.cu_history[0]: del self.cu_history[0][carId]
                if carId in self.cu_history[1]: del self.cu_history[1][carId]

    def calcRTInterval(self):
        with self.lock:
            res = self.intervals[0]
            if 0 < self.intervals[1] < res or res == 0:
                res = self.intervals[1]
            return res

    def setIntervals(self, pluginId, interval):
        with self.lock:
            oldInterval = self.calcRTInterval()
            self.intervals[pluginId] = interval
            newInterval = self.calcRTInterval()
            return newInterval

    def getInterval(self, pluginId):
        with self.lock:
            if self.intervals[pluginId] < 0:
                return None
            return self.intervals[pluginId]

    def infoRequest(self, pluginId, cls, f_filter):
        with self.lock:
            if len(self.info_requests) < 64:
                self.info_requests.append(self.InfoRequest(timestamp=time.time(), pluginId=pluginId, cls=cls, f_filter=f_filter))

    def okToSend(self, pluginId, packet):
        with self.lock:
            if type(packet) == ac_server_protocol.CarUpdate:
                # CarUpdate packets
                if self.intervals[pluginId] == 0:
                    # no rt report configured, CarUpdate event will not be passed
                    return False
                t = time.time()
                threshold = t - max(0, (self.intervals[pluginId]-50)*0.001)
                if not packet.carId in self.cu_history[pluginId]:
                    # create a history info object for the car if not already there
                    self.cu_history[pluginId][packet.carId] = self.HistoryInfo(lastSendTime = threshold, firstSendTime = t, count = 0)
                lastT = self.cu_history[pluginId][packet.carId].lastSendTime
                if t-lastT > self.intervals[pluginId]*10:
                    log_dbg("car %d has not been updated for a long time (the player probably left) - resetting statistics" % packet.carId)
                if ((self.intervals[pluginId] <= self.intervals[1-pluginId] or self.intervals[1-pluginId] <= 0) or
                    (lastT <= threshold)):
                    # this plugin has the quicker update rate
                    self.cu_history[pluginId][packet.carId].lastSendTime = t
                    self.cu_history[pluginId][packet.carId].count += 1
                    return True
                return False
            elif type(packet) in [ac_server_protocol.SessionInfo, ac_server_protocol.CarInfo]:
                # Requested info packets
                for ir in self.info_requests:
                    if ir.pluginId == pluginId:
                        if ir.cls == type(packet):
                            if ir.f_filter(packet):
                                self.info_requests.remove(ir)
                                return True
                        else:
                            pass
                # no request found for this packet. Probably already sent to proxy
                return False
            # generic packet. Needs proxying
            return True

    def plausibilityCheck(self):
        with self.lock:
            t = time.time()
            for ir in copy.copy(self.info_requests):
                if t - ir.timestamp > 5.:
                    self.info_requests.remove(ir)
                    if ir.pluginId == 0:
                        log=log_err
                    else:
                        log=log_dbg
                    log("Timeout [pluginId=%d] while waiting for request (%.1fs) for request %s." % (ir.pluginId, t-ir.timestamp, ir.cls))
            for pluginId in [0,1]:
                if pluginId == 0:
                    log=log_err
                else:
                    log=log_dbg
                if self.intervals[pluginId] > 0:
                    for carId in self.cu_history[pluginId].keys():
                        h = self.cu_history[pluginId][carId]
                        if h.count <= 5: continue
                        avgInterval = (h.lastSendTime - h.firstSendTime)/h.count*1000
                        if avgInterval > self.intervals[pluginId]*1.5 or avgInterval < self.intervals[pluginId]*0.5:
                            log("Realtime report interval mismatch [pluginId=%d, carId=%d]. Configured %d ms, measured %.1f ms." % (pluginId, carId, self.intervals[pluginId], avgInterval))
<commit_msg>- after reporting a realtime report interval mismatch, reset the statistics - define statistics window for realtime report to be 30s <commit_after>import copy, time
from threading import RLock
from . import ac_server_protocol
from .ac_server_helpers import DictToClass

class ACUdpMonitor:
    def __init__(self):
        # pluginId = 0 -> this plugin
        # pluginId = 1 -> proxied plugin
        self.HistoryInfo = DictToClass
        self.InfoRequest = DictToClass
        self.lock = RLock()
        self.reset()

    def reset(self, carId = None):
        with self.lock:
            if carId is None:
                self.intervals = [0,0]
                self.cu_history = [{},{}] # [pluginId][carId]
                self.info_requests = []
            else:
                if carId in self.cu_history[0]: del self.cu_history[0][carId]
                if carId in self.cu_history[1]: del self.cu_history[1][carId]

    def calcRTInterval(self):
        with self.lock:
            res = self.intervals[0]
            if 0 < self.intervals[1] < res or res == 0:
                res = self.intervals[1]
            return res

    def setIntervals(self, pluginId, interval):
        with self.lock:
            oldInterval = self.calcRTInterval()
            self.intervals[pluginId] = interval
            newInterval = self.calcRTInterval()
            return newInterval

    def getInterval(self, pluginId):
        with self.lock:
            if self.intervals[pluginId] < 0:
                return None
            return self.intervals[pluginId]

    def infoRequest(self, pluginId, cls, f_filter):
        with self.lock:
            if len(self.info_requests) < 64:
                self.info_requests.append(self.InfoRequest(timestamp=time.time(), pluginId=pluginId, cls=cls, f_filter=f_filter))

    def okToSend(self, pluginId, packet):
        with self.lock:
            if type(packet) == ac_server_protocol.CarUpdate:
                # CarUpdate packets
                if self.intervals[pluginId] == 0:
                    # no rt report configured, CarUpdate event will not be passed
                    return False
                t = time.time()
                threshold = t - max(0, (self.intervals[pluginId]-50)*0.001)
                if not packet.carId in self.cu_history[pluginId]:
                    # create a history info object for the car if not already there
                    self.cu_history[pluginId][packet.carId] = self.HistoryInfo(lastSendTime = threshold, firstSendTime = t, count = 0)
                lastT = self.cu_history[pluginId][packet.carId].lastSendTime
                if t-lastT > self.intervals[pluginId]*10:
                    log_dbg("car %d has not been updated for a long time (the player probably left) - resetting statistics" % packet.carId)
                    self.cu_history[pluginId][packet.carId] = self.HistoryInfo(lastSendTime = threshold, firstSendTime = t, count = 0)
                    lastT = threshold
                if ((self.intervals[pluginId] <= self.intervals[1-pluginId] or self.intervals[1-pluginId] <= 0) or
                    (lastT <= threshold)):
                    # this plugin has the quicker update rate
                    h = self.cu_history[pluginId][packet.carId]
                    h.lastSendTime = t
                    h.count += 1
                    # limit the history to 30s, intervals are in milliseconds
                    maxcnt = 30000./max(10,self.intervals[pluginId])
                    if h.count > maxcnt:
                        avg = (h.lastSendTime - h.firstSendTime)/h.count
                        h.count = maxcnt
                        h.firstSendTime = h.lastSendTime - avg*h.count
                    return True
                return False
            elif type(packet) in [ac_server_protocol.SessionInfo, ac_server_protocol.CarInfo]:
                # Requested info packets
                for ir in self.info_requests:
                    if ir.pluginId == pluginId:
                        if ir.cls == type(packet):
                            if ir.f_filter(packet):
                                self.info_requests.remove(ir)
                                return True
                        else:
                            pass
                # no request found for this packet. Probably already sent to proxy
                return False
            # generic packet. Needs proxying
            return True

    def plausibilityCheck(self):
        with self.lock:
            t = time.time()
            for ir in copy.copy(self.info_requests):
                if t - ir.timestamp > 5.:
                    self.info_requests.remove(ir)
                    if ir.pluginId == 0:
                        log=log_err
                    else:
                        log=log_dbg
                    log("Timeout [pluginId=%d] while waiting for request (%.1fs) for request %s." % (ir.pluginId, t-ir.timestamp, ir.cls))
            for pluginId in [0,1]:
                if pluginId == 0:
                    log=log_err
                else:
                    log=log_dbg
                if self.intervals[pluginId] > 0:
                    for carId in self.cu_history[pluginId].keys():
                        h = self.cu_history[pluginId][carId]
                        if h.count <= 10: continue
                        avgInterval = (h.lastSendTime - h.firstSendTime)/h.count*1000
                        if avgInterval > self.intervals[pluginId]*1.5 or avgInterval < self.intervals[pluginId]*0.5:
                            log("Realtime report interval mismatch [pluginId=%d, carId=%d]. Configured %d ms, measured %.1f ms. Resetting stats." % (pluginId, carId, self.intervals[pluginId], avgInterval))
                            del self.cu_history[pluginId][carId]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2009 Sharoon Thomas
#    Copyright (C) 2010-2010 OpenERP SA (<http://www.openerp.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from osv import osv, fields
from mako.template import Template
from mako import exceptions
import netsvc
import base64
from tools.translate import _
import tools
from email_template.email_template import get_value

## FIXME: this wizard duplicates a lot of features of the email template preview,
##        one of the 2 should inherit from the other!

class email_template_send_wizard(osv.osv_memory):
    _name = 'email_template.send.wizard'
    _description = 'This is the wizard for sending mail'
    _rec_name = "subject"

    def _get_accounts(self, cr, uid, context=None):
        if context is None:
            context = {}

        template = self._get_template(cr, uid, context)
        if not template:
            return []

        logger = netsvc.Logger()

        if template.from_account:
            return [(template.from_account.id, '%s (%s)' % (template.from_account.name, template.from_account.email_id))]
        else:
            account_id = self.pool.get('email.smtp_server').search(cr,uid,[('company','=','no'),('user','=',uid)], context=context)
            if account_id:
                account = self.pool.get('email.smtp_server').browse(cr,uid,account_id, context)
                return [(r.id,r.name + " (" + r.email_id + ")") for r in account]
            else:
                logger.notifyChannel(_("email-template"), netsvc.LOG_ERROR, _("No personal email accounts are configured for you. \nEither ask admin to enforce an account for this template or get yourself a personal email account."))
                raise osv.except_osv(_("Missing mail account"),_("No personal email accounts are configured for you. \nEither ask admin to enforce an account for this template or get yourself a personal email account."))

    def get_value(self, cursor, user, template, message, context=None, id=None):
        """Gets the value of the message parsed with the content of object id (or the first 'src_rec_ids' if id is not given)"""
        if not message:
            return ''
        if not id:
            id = context['src_rec_ids'][0]
        return get_value(cursor, user, id, message, template, context)

    def _get_template(self, cr, uid, context=None):
        if context is None:
            context = {}
        if not 'template' in context and not 'template_id' in context:
            return None
        if 'template_id' in context.keys():
            template_ids = self.pool.get('email.template').search(cr, uid, [('id','=',context['template_id'])], context=context)
        elif 'template' in context.keys():
            # Old versions of email_template used the name of the template. This caused
            # problems when the user changed the name of the template, but we keep the code
            # for compatibility with those versions.
            template_ids = self.pool.get('email.template').search(cr, uid, [('name','=',context['template'])], context=context)
        if not template_ids:
            return None

        template = self.pool.get('email.template').browse(cr, uid, template_ids[0], context)

        lang = self.get_value(cr, uid, template, template.lang, context)
        if lang:
            # Use translated template if necessary
            ctx = context.copy()
            ctx['lang'] = lang
            template = self.pool.get('email.template').browse(cr, uid, template.id, ctx)
        return template

    def _get_template_value(self, cr, uid, field, context=None):
        if context is None:
            context = {}
        template = self._get_template(cr, uid, context)
        if not template:
            return False
        if len(context['src_rec_ids']) > 1: # Multiple Mail: Gets original template values for multiple email change
            return getattr(template, field)
        else: # Simple Mail: Gets computed template values
            return self.get_value(cr, uid, template, getattr(template, field), context)

    _columns = {
        'state':fields.selection([
                        ('single','Simple Mail Wizard Step 1'),
                        ('multi','Multiple Mail Wizard Step 1'),
                        ('done','Wizard Complete')
                                  ],'Status',readonly=True),
        'ref_template':fields.many2one('email.template','Template',readonly=True),
        'rel_model':fields.many2one('ir.model','Model',readonly=True),
        'rel_model_ref':fields.integer('Referred Document',readonly=True),
        'from':fields.selection(_get_accounts,'From Account',select=True),
        'to':fields.char('To',size=250,required=True),
        'cc':fields.char('CC',size=250,),
        'bcc':fields.char('BCC',size=250,),
        'reply_to':fields.char('Reply-To',
                    size=250,
                    help="The address recipients should reply to,"
                         " if different from the From address."
                         " Placeholders can be used here."),
        'message_id':fields.char('Message-ID',
                    size=250,
                    help="The Message-ID header value, if you need to"
                         "specify it, for example to automatically recognize the replies later."
                        " Placeholders can be used here."),
        'subject':fields.char('Subject',size=200),
        'body_text':fields.text('Body',),
        'body_html':fields.text('Body',),
        'report':fields.char('Report File Name',size=100,),
        'signature':fields.boolean('Attach my signature to mail'),
        #'filename':fields.text('File Name'),
        'requested':fields.integer('No of requested Mails',readonly=True),
        'generated':fields.integer('No of generated Mails',readonly=True),
        'full_success':fields.boolean('Complete Success',readonly=True),
        'attachment_ids': fields.many2many('ir.attachment','send_wizard_attachment_rel', 'wizard_id', 'attachment_id', 'Attachments'),
    }

    #FIXME: probably better by overriding default_get directly
    _defaults = {
        'state': lambda self,cr,uid,ctx: len(ctx.get('src_rec_ids','')) > 1 and 'multi' or 'single',
        'rel_model': lambda self,cr,uid,ctx: self.pool.get('ir.model').search(cr,uid,[('model','=',ctx.get('src_model'))],context=ctx)[0],
        'rel_model_ref': lambda self,cr,uid,ctx: ctx['active_id'],
        'to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_to', ctx),
        'cc': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_cc', ctx),
        'bcc': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_bcc', ctx),
        'subject':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_subject', ctx),
        'body_text':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_body_text', ctx),
        'body_html':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_body_html', ctx),
        'report': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'file_name', ctx),
        'signature': lambda self,cr,uid,ctx: self._get_template(cr, uid, ctx).use_sign,
        'ref_template':lambda self,cr,uid,ctx: self._get_template(cr, uid, ctx).id,
        'reply_to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'reply_to', ctx),
        'reply_to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'reply_to', ctx),
        'requested':lambda self,cr,uid,ctx: len(ctx.get('src_rec_ids','')),
        'full_success': False,
        'attachment_ids': [],
    }

    def fields_get(self, cr, uid, fields=None, context=None, write_access=True):
        if context is None:
            context = {}
        result = super(email_template_send_wizard, self).fields_get(cr, uid, fields, context, write_access)
        if 'attachment_ids' in result and 'src_model' in context:
            result['attachment_ids']['domain'] = [('res_model','=',context['src_model']),('res_id','=',context['active_id'])]
        return result

    def sav_to_drafts(self, cr, uid, ids, context=None):
        if context is None:
            context = {}
        mailid = self.save_to_mailbox(cr, uid, ids, context=context)
        if self.pool.get('email.message').write(cr, uid, mailid, {'folder':'drafts'}, context):
            return {'type':'ir.actions.act_window_close' }

    def send_mail(self, cr, uid, ids, context=None):
        if context is None:
            context = {}
        mailid = self.save_to_mailbox(cr, uid, ids, context)
        if self.pool.get('email.message').write(cr, uid, mailid, {'folder':'outbox'}, context):
            return {'type':'ir.actions.act_window_close' }

    def get_generated(self, cr, uid, ids=None, context=None):
        if ids is None:
            ids = []
        if context is None:
            context = {}
        logger = netsvc.Logger()
        if context['src_rec_ids'] and len(context['src_rec_ids'])>1:
            #Means there are multiple items selected for email.
            mail_ids = self.save_to_mailbox(cr, uid, ids, context)
            if mail_ids:
                self.pool.get('email.message').write(cr, uid, mail_ids, {'folder':'outbox'}, context)
                logger.notifyChannel("email-template", netsvc.LOG_INFO, _("Emails for multiple items saved in outbox."))
                self.write(cr, uid, ids, {
                    'generated':len(mail_ids),
                    'state':'done'
                }, context)
            else:
                raise osv.except_osv(_("Email Template"),_("Email sending failed for one or more objects."))
        return True

    def save_to_mailbox(self, cr, uid, ids, context=None):
        def get_end_value(id, value):
            if len(context['src_rec_ids']) > 1: # Multiple Mail: Gets value from the template
                return self.get_value(cr, uid, template, value, context, id)
            else:
                return value

        if context is None:
            context = {}
        mail_ids = []
        email_message_obj = self.pool.get('email.message')
        template = self._get_template(cr, uid, context)
        for id in context['src_rec_ids']:
            screen_vals = self.read(cr, uid, ids[0], [], context)
            account = self.pool.get('email.smtp_server').read(cr, uid, screen_vals['from'], context=context)
            vals = {
                'email_from': tools.ustr(account['name']) + "<" + tools.ustr(account['email_id']) + ">",
                'email_to': get_end_value(id, screen_vals['to']),
                'email_cc': get_end_value(id, screen_vals['cc']),
                'email_bcc': get_end_value(id, screen_vals['bcc']),
                'subject': get_end_value(id, screen_vals['subject']),
                'body_text': get_end_value(id, screen_vals['body_text']),
                'body_html': get_end_value(id, screen_vals['body_html']),
                'account_id': screen_vals['from'],
                'state':'na',
                'mail_type':'multipart/alternative' #Options:'multipart/mixed','multipart/alternative','text/plain','text/html'
            }
            if screen_vals['signature']:
                signature = self.pool.get('res.users').read(cr, uid, uid, ['signature'], context)['signature']
                if signature:
                    vals['body_text'] = tools.ustr(vals['body_text'] or '') + signature
                    vals['body_html'] = tools.ustr(vals['body_html'] or '') + signature

            attachment_ids = []

            #Create partly the mail and later update attachments
            mail_id = email_message_obj.create(cr, uid, vals, context)
            mail_ids.append(mail_id)
            if template.report_template:
                reportname = 'report.' + self.pool.get('ir.actions.report.xml').read(cr, uid, template.report_template.id, ['report_name'], context)['report_name']
                data = {}
                data['model'] = self.pool.get('ir.model').browse(cr, uid, screen_vals['rel_model'], context).model

                # Ensure report is rendered using template's language
                ctx = context.copy()
                if template.lang:
                    ctx['lang'] = self.get_value(cr, uid, template, template.lang, context, id)
                service = netsvc.LocalService(reportname)
                (result, format) = service.create(cr, uid, [id], data, ctx)
                attachment_id = self.pool.get('ir.attachment').create(cr, uid, {
                    'name': _('%s (Email Attachment)') % tools.ustr(vals['subject']),
                    'datas': base64.b64encode(result),
                    'datas_fname': tools.ustr(get_end_value(id, screen_vals['report']) or _('Report')) + "." + format,
                    'description': vals['body_text'] or _("No Description"),
                    'res_model': 'email.message',
                    'res_id': mail_id
                }, context)
                attachment_ids.append( attachment_id )

            # Add document attachments
            for attachment_id in screen_vals.get('attachment_ids',[]):
                new_id = self.pool.get('ir.attachment').copy(cr, uid, attachment_id, {
                    'res_model': 'email.message',
                    'res_id': mail_id,
                }, context)
                attachment_ids.append( new_id )

            if attachment_ids:
                email_message_obj.write(cr, uid, mail_id, {
                    'attachments_ids': [[6, 0, attachment_ids]],
                    'mail_type': 'multipart/mixed'
                }, context)

        return mail_ids
email_template_send_wizard()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>Keep unused code in comment. Improve email_template_send wizard in email_template module.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2009 Sharoon Thomas
#    Copyright (C) 2010-2010 OpenERP SA (<http://www.openerp.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from osv import osv, fields
from mako.template import Template
from mako import exceptions
import netsvc
import base64
from tools.translate import _
import tools
from email_template.email_template import get_value

## FIXME: this wizard duplicates a lot of features of the email template preview,
##        one of the 2 should inherit from the other!

class email_template_send_wizard(osv.osv_memory):
    _name = 'email_template.send.wizard'
    _description = 'This is the wizard for sending mail'
    _rec_name = "subject"

#    def _get_accounts(self, cr, uid, context=None):
#        if context is None:
#            context = {}
#
#        template = self._get_template(cr, uid, context)
#        if not template:
#            return []
#
#        logger = netsvc.Logger()
#
#        if template.from_account:
#            return [(template.from_account.id, '%s (%s)' % (template.from_account.name, template.from_account.email_id))]
#        else:
#            account_id = self.pool.get('email.smtp_server').search(cr,uid,[('company','=','no'),('user','=',uid)], context=context)
#            if account_id:
#                account = self.pool.get('email.smtp_server').browse(cr,uid,account_id, context)
#                return [(r.id,r.name + " (" + r.email_id + ")") for r in account]
#            else:
#                logger.notifyChannel(_("email-template"), netsvc.LOG_ERROR, _("No personal email accounts are configured for you. \nEither ask admin to enforce an account for this template or get yourself a personal email account."))
#                raise osv.except_osv(_("Missing mail account"),_("No personal email accounts are configured for you. \nEither ask admin to enforce an account for this template or get yourself a personal email account."))

    def get_value(self, cursor, user, template, message, context=None, id=None):
        """Gets the value of the message parsed with the content of object id (or the first 'src_rec_ids' if id is not given)"""
        if not message:
            return ''
        if not id:
            id = context['src_rec_ids'][0]
        return get_value(cursor, user, id, message, template, context)

    def _get_template(self, cr, uid, context=None):
        if context is None:
            context = {}
        if not 'template' in context and not 'template_id' in context:
            return None
        if 'template_id' in context.keys():
            template_ids = self.pool.get('email.template').search(cr, uid, [('id','=',context['template_id'])], context=context)
        elif 'template' in context.keys():
            # Old versions of email_template used the name of the template. This caused
            # problems when the user changed the name of the template, but we keep the code
            # for compatibility with those versions.
            template_ids = self.pool.get('email.template').search(cr, uid, [('name','=',context['template'])], context=context)
        if not template_ids:
            return None

        template = self.pool.get('email.template').browse(cr, uid, template_ids[0], context)

        lang = self.get_value(cr, uid, template, template.lang, context)
        if lang:
            # Use translated template if necessary
            ctx = context.copy()
            ctx['lang'] = lang
            template = self.pool.get('email.template').browse(cr, uid, template.id, ctx)
        return template

    def _get_template_value(self, cr, uid, field, context=None):
        if context is None:
            context = {}
        template = self._get_template(cr, uid, context)
        if not template:
            return False
        if len(context['src_rec_ids']) > 1: # Multiple Mail: Gets original template values for multiple email change
            return getattr(template, field)
        else: # Simple Mail: Gets computed template values
            return self.get_value(cr, uid, template, getattr(template, field), context)

    _columns = {
        'state':fields.selection([
                        ('single','Simple Mail Wizard Step 1'),
                        ('multi','Multiple Mail Wizard Step 1'),
                        ('done','Wizard Complete')
                                  ],'Status',readonly=True),
        'ref_template':fields.many2one('email.template','Template',readonly=True),
        'rel_model':fields.many2one('ir.model','Model',readonly=True),
        'rel_model_ref':fields.integer('Referred Document',readonly=True),
#        'from':fields.selection(_get_accounts,'From Account',select=True),
        'to':fields.char('To',size=250,required=True),
        'cc':fields.char('CC',size=250,),
        'bcc':fields.char('BCC',size=250,),
        'reply_to':fields.char('Reply-To',
                    size=250,
                    help="The address recipients should reply to,"
                         " if different from the From address."
                         " Placeholders can be used here."),
        'message_id':fields.char('Message-ID',
                    size=250,
                    help="The Message-ID header value, if you need to"
                         "specify it, for example to automatically recognize the replies later."
                        " Placeholders can be used here."),
        'subject':fields.char('Subject',size=200),
        'body_text':fields.text('Body',),
        'body_html':fields.text('Body',),
        'report':fields.char('Report File Name',size=100,),
        'signature':fields.boolean('Attach my signature to mail'),
        #'filename':fields.text('File Name'),
        'requested':fields.integer('No of requested Mails',readonly=True),
        'generated':fields.integer('No of generated Mails',readonly=True),
        'full_success':fields.boolean('Complete Success',readonly=True),
        'attachment_ids': fields.many2many('ir.attachment','send_wizard_attachment_rel', 'wizard_id', 'attachment_id', 'Attachments'),
    }

    #FIXME: probably better by overriding default_get directly
    _defaults = {
        'state': lambda self,cr,uid,ctx: len(ctx.get('src_rec_ids','')) > 1 and 'multi' or 'single',
        'rel_model': lambda self,cr,uid,ctx: self.pool.get('ir.model').search(cr,uid,[('model','=',ctx.get('src_model'))],context=ctx)[0],
        'rel_model_ref': lambda self,cr,uid,ctx: ctx['active_id'],
        'to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_to', ctx),
        'cc': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_cc', ctx),
        'bcc': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_bcc', ctx),
        'subject':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_subject', ctx),
        'body_text':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_body_text', ctx),
        'body_html':lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'def_body_html', ctx),
        'report': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'file_name', ctx),
        'signature': lambda self,cr,uid,ctx: self._get_template(cr, uid, ctx).use_sign,
        'ref_template':lambda self,cr,uid,ctx: self._get_template(cr, uid, ctx).id,
        'reply_to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'reply_to', ctx),
        'reply_to': lambda self,cr,uid,ctx: self._get_template_value(cr, uid, 'reply_to', ctx),
        'requested':lambda self,cr,uid,ctx: len(ctx.get('src_rec_ids','')),
        'full_success': False,
        'attachment_ids': [],
    }

    def fields_get(self, cr, uid, fields=None, context=None, write_access=True):
        if context is None:
            context = {}
        result = super(email_template_send_wizard, self).fields_get(cr, uid, fields, context, write_access)
        if 'attachment_ids' in result and 'src_model' in context:
            result['attachment_ids']['domain'] = [('res_model','=',context['src_model']),('res_id','=',context['active_id'])]
        return result

    def sav_to_drafts(self, cr, uid, ids, context=None):
        if context is None:
            context = {}
        mailid = self.save_to_mailbox(cr, uid, ids, context=context)
        if self.pool.get('email.message').write(cr, uid, mailid, {'folder':'drafts'}, context):
            return {'type':'ir.actions.act_window_close' }

    def send_mail(self, cr, uid, ids, context=None):
        if context is None:
            context = {}
        mailid = self.save_to_mailbox(cr, uid, ids, context)
        if self.pool.get('email.message').write(cr, uid, mailid, {'folder':'outbox'}, context):
            return {'type':'ir.actions.act_window_close' }

    def get_generated(self, cr, uid, ids=None, context=None):
        if ids is None:
            ids = []
        if context is None:
            context = {}
        logger = netsvc.Logger()
        if context['src_rec_ids'] and len(context['src_rec_ids'])>1:
            #Means there are multiple items selected for email.
            mail_ids = self.save_to_mailbox(cr, uid, ids, context)
            if mail_ids:
                self.pool.get('email.message').write(cr, uid, mail_ids, {'folder':'outbox'}, context)
                logger.notifyChannel("email-template", netsvc.LOG_INFO, _("Emails for multiple items saved in outbox."))
                self.write(cr, uid, ids, {
                    'generated':len(mail_ids),
                    'state':'done'
                }, context)
            else:
                raise osv.except_osv(_("Email Template"),_("Email sending failed for one or more objects."))
        return True

    def save_to_mailbox(self, cr, uid, ids, context=None):
        def get_end_value(id, value):
            if len(context['src_rec_ids']) > 1: # Multiple Mail: Gets value from the template
                return self.get_value(cr, uid, template, value, context, id)
            else:
                return value

        if context is None:
            context = {}
        mail_ids = []
        email_message_obj = self.pool.get('email.message')
        template = self._get_template(cr, uid, context)
        for id in context['src_rec_ids']:
            screen_vals = self.read(cr, uid, ids[0], [], context)
            account = self.pool.get('email.smtp_server').read(cr, uid, screen_vals['from'], context=context)
            vals = {
                'email_from': tools.ustr(account['name']) + "<" + tools.ustr(account['email_id']) + ">",
                'email_to': get_end_value(id, screen_vals['to']),
                'email_cc': get_end_value(id, screen_vals['cc']),
                'email_bcc': get_end_value(id, screen_vals['bcc']),
                'subject': get_end_value(id, screen_vals['subject']),
                'body_text': get_end_value(id, screen_vals['body_text']),
                'body_html': get_end_value(id, screen_vals['body_html']),
#                'account_id': screen_vals['from'],
                'state':'na',
                'mail_type':'multipart/alternative' #Options:'multipart/mixed','multipart/alternative','text/plain','text/html'
            }
            if screen_vals['signature']:
                signature = self.pool.get('res.users').read(cr, uid, uid, ['signature'], context)['signature']
                if signature:
                    vals['body_text'] = tools.ustr(vals['body_text'] or '') + signature
                    vals['body_html'] = tools.ustr(vals['body_html'] or '') + signature

            attachment_ids = []

            #Create partly the mail and later update attachments
            mail_id = email_message_obj.create(cr, uid, vals, context)
            mail_ids.append(mail_id)
            if template.report_template:
                reportname = 'report.' + self.pool.get('ir.actions.report.xml').read(cr, uid, template.report_template.id, ['report_name'], context)['report_name']
                data = {}
                data['model'] = self.pool.get('ir.model').browse(cr, uid, screen_vals['rel_model'], context).model

                # Ensure report is rendered using template's language
                ctx = context.copy()
                if template.lang:
                    ctx['lang'] = self.get_value(cr, uid, template, template.lang, context, id)
                service = netsvc.LocalService(reportname)
                (result, format) = service.create(cr, uid, [id], data, ctx)
                attachment_id = self.pool.get('ir.attachment').create(cr, uid, {
                    'name': _('%s (Email Attachment)') % tools.ustr(vals['subject']),
                    'datas': base64.b64encode(result),
                    'datas_fname': tools.ustr(get_end_value(id, screen_vals['report']) or _('Report')) + "." + format,
                    'description': vals['body_text'] or _("No Description"),
                    'res_model': 'email.message',
                    'res_id': mail_id
                }, context)
                attachment_ids.append( attachment_id )

            # Add document attachments
            for attachment_id in screen_vals.get('attachment_ids',[]):
                new_id = self.pool.get('ir.attachment').copy(cr, uid, attachment_id, {
                    'res_model': 'email.message',
                    'res_id': mail_id,
                }, context)
                attachment_ids.append( new_id )

            if attachment_ids:
                email_message_obj.write(cr, uid, mail_id, {
                    'attachments_ids': [[6, 0, attachment_ids]],
                    'mail_type': 'multipart/mixed'
                }, context)

        return mail_ids
email_template_send_wizard()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>import urllib3
import io
import gzip
import csv
import logging
import pprint
from pyramid.view import view_config
from elasticsearch.exceptions import (
    NotFoundError
)
from contentbase import DBSESSION
from contentbase.storage import (
    TransactionRecord,
)
from contentbase.elasticsearch.indexer import all_uuids
from contentbase.elasticsearch.interfaces import (
    ELASTIC_SEARCH,
    SNP_SEARCH_ES,
)

from random import shuffle

SEARCH_MAX = 99999  # OutOfMemoryError if too high
log = logging.getLogger(__name__)

# hashmap of assays and corresponding file types that are being indexed
_INDEXED_DATA = {
    'ChIP-seq': {
        'output_type': ['optimal idr thresholded peaks'],
    },
    'DNase-seq': {
        'file_type': ['bed narrowPeak']
    },
    'eCLIP': {
        'file_type': ['bed narrowPeak']
    }
}

# Species and references being indexed
_SPECIES = {
    'Homo sapiens': ['hg19']
}

def includeme(config):
    config.add_route('index_file', '/index_file')
    config.scan(__name__)

def tsvreader(file):
    reader = csv.reader(file, delimiter='\t')
    for row in reader:
        yield row

# Mapping should be generated dynamically for each assembly type
def get_mapping(assembly_name='hg19'):
    return {
        assembly_name: {
            '_all': {
                'enabled': False
            },
            '_source': {
                'enabled': False
            },
            'properties': {
                'uuid': {
                    'type': 'string',
                    'index': 'not_analyzed'
                },
                'positions': {
                    'type': 'nested',
                    'properties': {
                        'start': {
                            'type': 'long'
                        },
                        'end': {
                            'type': 'long'
                        }
                    }
                }
            }
        }
    }

def index_settings():
    return {
        'index': {
            'number_of_shards': 1
        }
    }

def get_assay_term_name(accession, request):
    '''
    Input file accession and returns assay_term_name of the experiment the file
    belongs to
    '''
    context = request.embed(accession)
    if 'assay_term_name' in context:
        return context['assay_term_name']
    return None

def index_peaks(uuid, request):
    """
    Indexes bed files in elasticsearch index
    """

    context = request.embed(uuid)

    urllib3.disable_warnings()
    es = request.registry.get(SNP_SEARCH_ES, None)
    http = urllib3.PoolManager()
    r = http.request('GET', 'https://www.encodeproject.org/files/ENCFF558TCP/@@download/ENCFF558TCP.bed.gz')
    #r = http.request('GET', request.host_url + output_file['href'])

    comp = io.BytesIO()
    comp.write(r.data)
    comp.seek(0)
    r.release_conn()
    file_data = dict()

    with gzip.open(comp, mode='rt') as file:
        for row in tsvreader(file):
            chrom, start, end = row[0].lower(), int(row[1]), int(row[2])
            if isinstance(start, int) and isinstance(end, int):
                if chrom in file_data:
                    file_data[chrom].append({
                        'start': start + 1,
                        'end': end + 1
                    })
                else:
                    file_data[chrom] = [{'start': start + 1, 'end': end + 1}]

    log.warn("file successfully read for indexing")

    for key in file_data:
        doc = {
            'uuid': output_file['uuid'],
            'positions': file_data[key]
        }
        if not es.indices.exists(key):
            es.indices.create(index=key, body=index_settings())
            es.indices.put_mapping(index=key, doc_type=output_file['assembly'],
                                   body=get_mapping(output_file['assembly']))
        es.index(index=key, doc_type=output_file['assembly'], body=doc,
                 id=output_file['uuid'])

    if 'AnalysisStepRun' not in context['@type']:
        return

    for output_file in context['output_files']:

        if 'File' not in output_file['@type'] or 'dataset' not in output_file:
            continue

        if 'status' not in output_file and output_file['status'] != 'released':
            continue

        # Index human data only for now
        # if 'hg19' not in output_file['assembly']:
        #     continue

        assay_term_name = get_assay_term_name(output_file['dataset'], request)
        if assay_term_name is None:
            log.warn("assay_term_name is none")
            continue

        flag = False

        for k, v in _INDEXED_DATA.get(assay_term_name, {}).items():
            if k in output_file and output_file[k] in v:
                if 'file_format' in output_file and output_file['file_format'] == 'bed':
                    flag = True
                    break
        if not flag:
            continue

        log.warn("qualifying bed file found")

        urllib3.disable_warnings()
        es = request.registry.get(SNP_SEARCH_ES, None)
        http = urllib3.PoolManager()
        r = http.request('GET', 'https://www.encodeproject.org/files/ENCFF558TCP/@@download/ENCFF558TCP.bed.gz')
        #r = http.request('GET', request.host_url + output_file['href'])
        if r.status != 200:
            continue
        comp = io.BytesIO()
        comp.write(r.data)
        comp.seek(0)
        r.release_conn()
        file_data = dict()

        with gzip.open(comp, mode='rt') as file:
            for row in tsvreader(file):
                chrom, start, end = row[0].lower(), int(row[1]), int(row[2])
                if isinstance(start, int) and isinstance(end, int):
                    if chrom in file_data:
                        file_data[chrom].append({
                            'start': start + 1,
                            'end': end + 1
                        })
                    else:
                        file_data[chrom] = [{'start': start + 1, 'end': end + 1}]

        log.warn("file successfully read for indexing")

        for key in file_data:
            doc = {
                'uuid': output_file['uuid'],
                'positions': file_data[key]
            }
            if not es.indices.exists(key):
                es.indices.create(index=key, body=index_settings())
                es.indices.put_mapping(index=key, doc_type=output_file['assembly'],
                                       body=get_mapping(output_file['assembly']))
            es.index(index=key, doc_type=output_file['assembly'], body=doc,
                     id=output_file['uuid'])

@view_config(route_name='index_file', request_method='POST', permission="index")
def index_file(request):
    log.warn('Peak indexer started')
    INDEX = request.registry.settings['contentbase.elasticsearch.index']
    request.datastore = 'database'
    dry_run = request.json.get('dry_run', False)
    recovery = request.json.get('recovery', False)
    es = request.registry[ELASTIC_SEARCH]

    session = request.registry[DBSESSION]()
    connection = session.connection()
    if recovery:
        query = connection.execute(
            "SET TRANSACTION ISOLATION LEVEL READ COMMITTED, READ ONLY;"
            "SELECT txid_snapshot_xmin(txid_current_snapshot());"
        )
    else:
        query = connection.execute(
            "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE, READ ONLY, DEFERRABLE;"
            "SELECT txid_snapshot_xmin(txid_current_snapshot());"
        )
    xmin = query.scalar()  # lowest xid that is still in progress

    first_txn = None
    last_xmin = None
    if 'last_xmin' in request.json:
        last_xmin = request.json['last_xmin']
    else:
        try:
            status = es.get(index=INDEX, doc_type='meta', id='indexing')
        except NotFoundError:
            pass
        else:
            last_xmin = status['_source']['xmin']

    result = {
        'xmin': xmin,
        'last_xmin': last_xmin,
    }

    last_xmin = None    # overriding the varible to trigger indexing and avoid _indexer's interference

    if last_xmin is None:
        result['types'] = types = request.json.get('types', None)
        invalidated = list(all_uuids(request.root, types))
    else:
        txns = session.query(TransactionRecord).filter(
            TransactionRecord.xid >= last_xmin,
        )

        invalidated = set()
        updated = set()
        renamed = set()
        max_xid = 0
        txn_count = 0
        for txn in txns.all():
            txn_count += 1
            max_xid = max(max_xid, txn.xid)
            if first_txn is None:
                first_txn = txn.timestamp
            else:
                first_txn = min(first_txn, txn.timestamp)
            renamed.update(txn.data.get('renamed', ()))
            updated.update(txn.data.get('updated', ()))

        result['txn_count'] = txn_count
        if txn_count == 0:
            return result

        es.indices.refresh(index=INDEX)
        res = es.search(index=INDEX, size=SEARCH_MAX, body={
            'filter': {
                'or': [
                    {
                        'terms': {
                            'embedded_uuids': updated,
                            '_cache': False,
                        },
                    },
                    {
                        'terms': {
                            'linked_uuids': renamed,
                            '_cache': False,
                        },
                    },
                ],
            },
            '_source': False,
        })
        if res['hits']['total'] > SEARCH_MAX:
            invalidated = list(all_uuids(request.root))
        else:
            referencing = {hit['_id'] for hit in res['hits']['hits']}
            invalidated = referencing | updated
    if not dry_run:

        log.warn("Number of invalidated objects {}".format(len(invalidated)))

        shuffle(invalidated)
        for uuid in invalidated: # shuffling to see differt types of files in logs
            index_peaks(uuid, request)
    return result
<commit_msg>file download successful and gzip does throw ann exeption<commit_after>import urllib3
import io
import gzip
import csv
import logging
import pprint
from pyramid.view import view_config
from elasticsearch.exceptions import (
    NotFoundError
)
from contentbase import DBSESSION
from contentbase.storage import (
    TransactionRecord,
)
from contentbase.elasticsearch.indexer import all_uuids
from contentbase.elasticsearch.interfaces import (
    ELASTIC_SEARCH,
    SNP_SEARCH_ES,
)

from random import shuffle

SEARCH_MAX = 99999  # OutOfMemoryError if too high
log = logging.getLogger(__name__)

# hashmap of assays and corresponding file types that are being indexed
_INDEXED_DATA = {
    'ChIP-seq': {
        'output_type': ['optimal idr thresholded peaks'],
    },
    'DNase-seq': {
        'file_type': ['bed narrowPeak']
    },
    'eCLIP': {
        'file_type': ['bed narrowPeak']
    }
}

# Species and references being indexed
_SPECIES = {
    'Homo sapiens': ['hg19']
}

def includeme(config):
    config.add_route('index_file', '/index_file')
    config.scan(__name__)

def tsvreader(file):
    reader = csv.reader(file, delimiter='\t')
    for row in reader:
        yield row

# Mapping should be generated dynamically for each assembly type
def get_mapping(assembly_name='hg19'):
    return {
        assembly_name: {
            '_all': {
                'enabled': False
            },
            '_source': {
                'enabled': False
            },
            'properties': {
                'uuid': {
                    'type': 'string',
                    'index': 'not_analyzed'
                },
                'positions': {
                    'type': 'nested',
                    'properties': {
                        'start': {
                            'type': 'long'
                        },
                        'end': {
                            'type': 'long'
                        }
                    }
                }
            }
        }
    }

def index_settings():
    return {
        'index': {
            'number_of_shards': 1
        }
    }

def get_assay_term_name(accession, request):
    '''
    Input file accession and returns assay_term_name of the experiment the file
    belongs to
    '''
    context = request.embed(accession)
    if 'assay_term_name' in context:
        return context['assay_term_name']
    return None

def index_peaks(uuid, request):
    """
    Indexes bed files in elasticsearch index
    """

    context = request.embed(uuid)

    if 'AnalysisStepRun' not in context['@type']:
        return

    for output_file in context['output_files']:

        if 'File' not in output_file['@type'] or 'dataset' not in output_file:
            continue

        if 'status' not in output_file and output_file['status'] != 'released':
            continue

        # Index human data only for now
        # if 'hg19' not in output_file['assembly']:
        #     continue

        assay_term_name = get_assay_term_name(output_file['dataset'], request)
        if assay_term_name is None:
            continue

        flag = False

        for k, v in _INDEXED_DATA.get(assay_term_name, {}).items():
            if k in output_file and output_file[k] in v:
                if 'file_format' in output_file and output_file['file_format'] == 'bed':
                    flag = True
                    break
        if not flag:
            continue

        log.warn("qualifying bed file found")

        urllib3.disable_warnings()
        es = request.registry.get(SNP_SEARCH_ES, None)
        http = urllib3.PoolManager()
        r = http.request('GET', request.host_url + output_file['href'])
        if r.status != 200:
            continue
        comp = io.BytesIO()
        comp.write(r.data)
        comp.seek(0)
        r.release_conn()
        file_data = dict()

        with gzip.open(comp, mode='rt') as file:
            for row in tsvreader(file):
                chrom, start, end = row[0].lower(), int(row[1]), int(row[2])
                if isinstance(start, int) and isinstance(end, int):
                    if chrom in file_data:
                        file_data[chrom].append({
                            'start': start + 1,
                            'end': end + 1
                        })
                    else:
                        file_data[chrom] = [{'start': start + 1, 'end': end + 1}]

        for key in file_data:
            doc = {
                'uuid': output_file['uuid'],
                'positions': file_data[key]
            }
            if not es.indices.exists(key):
                es.indices.create(index=key, body=index_settings())
                es.indices.put_mapping(index=key, doc_type=output_file['assembly'],
                                       body=get_mapping(output_file['assembly']))
            es.index(index=key, doc_type=output_file['assembly'], body=doc,
                     id=output_file['uuid'])
        log.warn("bed file was indexed")

@view_config(route_name='index_file', request_method='POST', permission="index")
def index_file(request):
    log.warn('Peak indexer started')
    INDEX = request.registry.settings['contentbase.elasticsearch.index']
    request.datastore = 'database'
    dry_run = request.json.get('dry_run', False)
    recovery = request.json.get('recovery', False)
    es = request.registry[ELASTIC_SEARCH]

    session = request.registry[DBSESSION]()
    connection = session.connection()
    if recovery:
        query = connection.execute(
            "SET TRANSACTION ISOLATION LEVEL READ COMMITTED, READ ONLY;"
            "SELECT txid_snapshot_xmin(txid_current_snapshot());"
        )
    else:
        query = connection.execute(
            "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE, READ ONLY, DEFERRABLE;"
            "SELECT txid_snapshot_xmin(txid_current_snapshot());"
        )
    xmin = query.scalar()  # lowest xid that is still in progress

    first_txn = None
    last_xmin = None
    if 'last_xmin' in request.json:
        last_xmin = request.json['last_xmin']
    else:
        try:
            status = es.get(index=INDEX, doc_type='meta', id='indexing')
        except NotFoundError:
            pass
        else:
            last_xmin = status['_source']['xmin']

    result = {
        'xmin': xmin,
        'last_xmin': last_xmin,
    }

    if last_xmin is None:
        result['types'] = types = request.json.get('types', None)
        invalidated = list(all_uuids(request.root, types))
    else:
        txns = session.query(TransactionRecord).filter(
            TransactionRecord.xid >= last_xmin,
        )

        invalidated = set()
        updated = set()
        renamed = set()
        max_xid = 0
        txn_count = 0
        for txn in txns.all():
            txn_count += 1
            max_xid = max(max_xid, txn.xid)
            if first_txn is None:
                first_txn = txn.timestamp
            else:
                first_txn = min(first_txn, txn.timestamp)
            renamed.update(txn.data.get('renamed', ()))
            updated.update(txn.data.get('updated', ()))

        result['txn_count'] = txn_count
        if txn_count == 0:
            return result

        es.indices.refresh(index=INDEX)
        res = es.search(index=INDEX, size=SEARCH_MAX, body={
            'filter': {
                'or': [
                    {
                        'terms': {
                            'embedded_uuids': updated,
                            '_cache': False,
                        },
                    },
                    {
                        'terms': {
                            'linked_uuids': renamed,
                            '_cache': False,
                        },
                    },
                ],
            },
            '_source': False,
        })
        if res['hits']['total'] > SEARCH_MAX:
            invalidated = list(all_uuids(request.root))
        else:
            referencing = {hit['_id'] for hit in res['hits']['hits']}
            invalidated = referencing | updated
    if not dry_run:

        log.warn("Number of invalidated objects {}".format(len(invalidated)))

        shuffle(invalidated)
        for uuid in invalidated: # shuffling to see differt types of files in logs
            index_peaks(uuid, request)
    return result
<|endoftext|>