<commit_before>import time
import numpy as np
import scipy.sparse as sp
import os
import json
from scipy.constants import mu_0

import discretize
from discretize import utils
import properties
from SimPEG.EM import FDEM, TDEM
from SimPEG import Utils, Maps
from SimPEG.EM.Static import DC

try:
    from pymatsolver import Pardiso as Solver
except ImportError:
    import warnings
    warnings.warn(
        "Could not import Pardiso, falling back to LU. Will be slow."
    )
    from SimPEG import SolverLU as Solver

from .base import LoadableInstance, BaseCasing
from . import model
from .model import PhysicalProperties
from .mesh import BaseMeshGenerator, CylMeshGenerator, TensorMeshGenerator
from .sources import BaseCasingSrc
from .utils import writeSimulationPy
from . import sources
from .info import __version__

class BaseSimulation(BaseCasing):
    """
    Base class wrapper to run an EM Forward Simulation
    """

    fields_filename = properties.String(
        "filename for the fields",
        default="fields.npy"
    )

    filename = properties.String(
        "filename for the simulation parameters",
        default="simulationParameters.json"
    )

    num_threads = properties.Integer(
        "number of threads",
        default=1
    )

    modelParameters = LoadableInstance(
        "Model Parameters instance",
        model.Wholespace,
        required=True
    )

    meshGenerator = LoadableInstance(
        "mesh generator instance",
        BaseMeshGenerator,
        required=True
    )

    src = LoadableInstance(
        "Source Parameters instance",
        BaseCasingSrc,
        required=False
    )

    def __init__(self, **kwargs):
        # set keyword arguments
        Utils.setKwargs(self, **kwargs)

        # if the working directory does not exsist, create it
        if not os.path.isdir(self.directory):
            os.mkdir(self.directory)

        # hook up the properties classes
        self.meshGenerator.modelParameters = self.modelParameters

        if getattr(self, 'src', None) is not None:
            self.src.modelParameters = self.modelParameters
            self.src.meshGenerator = self.meshGenerator

    @property
    def physprops(self):
        if getattr(self, '_physprops', None) is None:
            self._physprops = PhysicalProperties(
                self.meshGenerator, self.modelParameters
            )
        return self._physprops

    @property
    def prob(self):
        return self._prob

    @property
    def survey(self):
        return self._survey

    def write_py(self, physics=None, includeDC=True, include2D=True):
        """
        Write a python script for running the simulation
        :param str physics: 'TDEM', 'FDEM'
        :param bool includeDC: include a DC simulation with the EM one (default is True)
        :param bool include2D: include a 2D simulation? (default is True)
        """

        # save the properties
        self.modelParameters.save()
        self.meshGenerator.save()
        self.src.save()

        # if physics is None:
        #     physics = self.src.physics

        # write the simulation.py
        writeSimulationPy(
            modelParameters=self.modelParameters.filename,
            meshGenerator=self.meshGenerator.filename,
            src=self.src.filename,
            directory=self.directory,
            physics=self.physics
        )

    def fields(self):
        """
        fields from the forward simulation
        """
        if getattr(self, '_fields', None) is None:
            self._fields = self.run()
        return self._fields

    def run(self):
        """
        Run the forward simulation
        """

        # ----------------- Validate Parameters ----------------- #

        print('Validating parameters...')
        self.validate()

        # grab the discretize mesh off of the mesh object
        sim_mesh = self.meshGenerator.mesh
        print('      max x: {}, min z: {}, max z: {}'.format(
            sim_mesh.vectorNx.max(),
            sim_mesh.vectorNz.min(),
            sim_mesh.vectorNz.max()
        ))

        # save simulation parameters
        self.save()

        # ----------------- Set up the simulation ----------------- #
        physprops = self.physprops
        prb = self.prob
        # survey = self.survey
        # prb.pair(survey)

        # ----------------- Run the the simulation ----------------- #
        print('Starting Simulation')
        t = time.time()
        print('Using {} Solver'.format(prb.Solver))
        fields = prb.fields(physprops.model)
        np.save(
            '/'.join([self.directory, self.fields_filename]),
            fields[:, '{}Solution'.format(self.formulation)]
        )
        print('   ... Done. Elapsed time : {}'.format(time.time()-t))

        self._fields = fields
        return fields

class SimulationFDEM(BaseSimulation):
    """
    A wrapper to run an FDEM Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """

    formulation = properties.StringChoice(
        "Formulation of the problem to solve [e, b, h, j]",
        default="h",
        choices=["e", "b", "h", "j"]
    )

    physics = "FDEM"

    def __init__(self, **kwargs):
        super(SimulationFDEM, self).__init__(**kwargs)

        self._prob = getattr(
                FDEM, 'Problem3D_{}'.format(self.formulation)
                )(
                self.meshGenerator.mesh,
                sigmaMap=self.physprops.wires.sigma,
                muMap=self.physprops.wires.mu,
                Solver=Solver,
                verbose=True
            )

        if getattr(self.src, "physics", None) is None:
            self.src.physics = "FDEM"

        self._survey = FDEM.Survey(self.src.srcList)

        self._prob.pair(self._survey)

class SimulationTDEM(BaseSimulation):
    """
    A wrapper to run a TDEM Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """

    formulation = properties.StringChoice(
        "Formulation of the problem to solve [e, b, h, j]",
        default="j",
        choices=["e", "b", "h", "j"]
    )

    physics = "TDEM"

    def __init__(self, **kwargs):
        super(SimulationTDEM, self).__init__(**kwargs)

        self._prob = getattr(
                TDEM, 'Problem3D_{}'.format(self.formulation)
                )(
                self.meshGenerator.mesh,
                timeSteps=self.modelParameters.timeSteps,
                sigmaMap=self.physprops.wires.sigma,
                mu=self.physprops.mu, # right now the TDEM code doesn't support mu inversions
                Solver=Solver,
                verbose=True
            )

        if getattr(self.src, "physics", None) is None:
            self.src.physics = "TDEM"

        self._survey = TDEM.Survey(self.src.srcList)

        self._prob.pair(self._survey)

class SimulationDC(BaseSimulation):
    """
    A wrapper to run a DC Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """
    src_a = properties.Vector3(
        "a electrode location", required=True
    )

    src_b = properties.Vector3(
        "return electrode location", required=True
    )

    fields_filename = properties.String(
        "filename for the fields",
        default="fieldsDC.npy"
    )

    formulation = properties.String(
        "field that we are solving for",
        default="phi"
    )

    physics = "DC"

    def __init__(self, **kwargs):
        super(SimulationDC, self).__init__(**kwargs)

        self._prob = DC.Problem3D_CC(
            self.meshGenerator.mesh,
            sigmaMap=self.physprops.wires.sigma,
            bc_type='Dirichlet',
            Solver=Solver
        )
        self._src = DC.Src.Dipole([], self.src_a, self.src_b)
        self._survey = DC.Survey([self._src])

        self._prob.pair(self._survey)
<commit_msg>print out number of cells<commit_after>import time
import numpy as np
import scipy.sparse as sp
import os
import json
from scipy.constants import mu_0

import discretize
from discretize import utils
import properties
from SimPEG.EM import FDEM, TDEM
from SimPEG import Utils, Maps
from SimPEG.EM.Static import DC

try:
    from pymatsolver import Pardiso as Solver
except ImportError:
    import warnings
    warnings.warn(
        "Could not import Pardiso, falling back to LU. Will be slow."
    )
    from SimPEG import SolverLU as Solver

from .base import LoadableInstance, BaseCasing
from . import model
from .model import PhysicalProperties
from .mesh import BaseMeshGenerator, CylMeshGenerator, TensorMeshGenerator
from .sources import BaseCasingSrc
from .utils import writeSimulationPy
from . import sources
from .info import __version__

class BaseSimulation(BaseCasing):
    """
    Base class wrapper to run an EM Forward Simulation
    """

    fields_filename = properties.String(
        "filename for the fields",
        default="fields.npy"
    )

    filename = properties.String(
        "filename for the simulation parameters",
        default="simulationParameters.json"
    )

    num_threads = properties.Integer(
        "number of threads",
        default=1
    )

    modelParameters = LoadableInstance(
        "Model Parameters instance",
        model.Wholespace,
        required=True
    )

    meshGenerator = LoadableInstance(
        "mesh generator instance",
        BaseMeshGenerator,
        required=True
    )

    src = LoadableInstance(
        "Source Parameters instance",
        BaseCasingSrc,
        required=False
    )

    def __init__(self, **kwargs):
        # set keyword arguments
        Utils.setKwargs(self, **kwargs)

        # if the working directory does not exsist, create it
        if not os.path.isdir(self.directory):
            os.mkdir(self.directory)

        # hook up the properties classes
        self.meshGenerator.modelParameters = self.modelParameters

        if getattr(self, 'src', None) is not None:
            self.src.modelParameters = self.modelParameters
            self.src.meshGenerator = self.meshGenerator

    @property
    def physprops(self):
        if getattr(self, '_physprops', None) is None:
            self._physprops = PhysicalProperties(
                self.meshGenerator, self.modelParameters
            )
        return self._physprops

    @property
    def prob(self):
        return self._prob

    @property
    def survey(self):
        return self._survey

    def write_py(self, physics=None, includeDC=True, include2D=True):
        """
        Write a python script for running the simulation
        :param str physics: 'TDEM', 'FDEM'
        :param bool includeDC: include a DC simulation with the EM one (default is True)
        :param bool include2D: include a 2D simulation? (default is True)
        """

        # save the properties
        self.modelParameters.save()
        self.meshGenerator.save()
        self.src.save()

        # if physics is None:
        #     physics = self.src.physics

        # write the simulation.py
        writeSimulationPy(
            modelParameters=self.modelParameters.filename,
            meshGenerator=self.meshGenerator.filename,
            src=self.src.filename,
            directory=self.directory,
            physics=self.physics
        )

    def fields(self):
        """
        fields from the forward simulation
        """
        if getattr(self, '_fields', None) is None:
            self._fields = self.run()
        return self._fields

    def run(self):
        """
        Run the forward simulation
        """

        # ----------------- Validate Parameters ----------------- #

        print('Validating parameters...')
        self.validate()

        # grab the discretize mesh off of the mesh object
        sim_mesh = self.meshGenerator.mesh
        print('      max x: {}, min z: {}, max z: {}, nC: {}'.format(
            sim_mesh.vectorNx.max(),
            sim_mesh.vectorNz.min(),
            sim_mesh.vectorNz.max(),
            sim_mesh.nC
        ))

        # save simulation parameters
        self.save()

        # ----------------- Set up the simulation ----------------- #
        physprops = self.physprops
        prb = self.prob
        # survey = self.survey
        # prb.pair(survey)

        # ----------------- Run the the simulation ----------------- #
        print('Starting Simulation')
        t = time.time()
        print('Using {} Solver'.format(prb.Solver))
        fields = prb.fields(physprops.model)
        np.save(
            '/'.join([self.directory, self.fields_filename]),
            fields[:, '{}Solution'.format(self.formulation)]
        )
        print('   ... Done. Elapsed time : {}'.format(time.time()-t))

        self._fields = fields
        return fields

class SimulationFDEM(BaseSimulation):
    """
    A wrapper to run an FDEM Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """

    formulation = properties.StringChoice(
        "Formulation of the problem to solve [e, b, h, j]",
        default="h",
        choices=["e", "b", "h", "j"]
    )

    physics = "FDEM"

    def __init__(self, **kwargs):
        super(SimulationFDEM, self).__init__(**kwargs)

        self._prob = getattr(
                FDEM, 'Problem3D_{}'.format(self.formulation)
                )(
                self.meshGenerator.mesh,
                sigmaMap=self.physprops.wires.sigma,
                muMap=self.physprops.wires.mu,
                Solver=Solver,
                verbose=True
            )

        if getattr(self.src, "physics", None) is None:
            self.src.physics = "FDEM"

        self._survey = FDEM.Survey(self.src.srcList)

        self._prob.pair(self._survey)

class SimulationTDEM(BaseSimulation):
    """
    A wrapper to run a TDEM Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """

    formulation = properties.StringChoice(
        "Formulation of the problem to solve [e, b, h, j]",
        default="j",
        choices=["e", "b", "h", "j"]
    )

    physics = "TDEM"

    def __init__(self, **kwargs):
        super(SimulationTDEM, self).__init__(**kwargs)

        self._prob = getattr(
                TDEM, 'Problem3D_{}'.format(self.formulation)
                )(
                self.meshGenerator.mesh,
                timeSteps=self.modelParameters.timeSteps,
                sigmaMap=self.physprops.wires.sigma,
                mu=self.physprops.mu, # right now the TDEM code doesn't support mu inversions
                Solver=Solver,
                verbose=True
            )

        if getattr(self.src, "physics", None) is None:
            self.src.physics = "TDEM"

        self._survey = TDEM.Survey(self.src.srcList)

        self._prob.pair(self._survey)

class SimulationDC(BaseSimulation):
    """
    A wrapper to run a DC Forward Simulation
    :param CasingSimulations.model.WholeSpace modelParameters: casing parameters object
    :param CasingSimulations.mesh.BaseMeshGenerator mesh: a CasingSimulation mesh generator object
    """
    src_a = properties.Vector3(
        "a electrode location", required=True
    )

    src_b = properties.Vector3(
        "return electrode location", required=True
    )

    fields_filename = properties.String(
        "filename for the fields",
        default="fieldsDC.npy"
    )

    formulation = properties.String(
        "field that we are solving for",
        default="phi"
    )

    physics = "DC"

    def __init__(self, **kwargs):
        super(SimulationDC, self).__init__(**kwargs)

        self._prob = DC.Problem3D_CC(
            self.meshGenerator.mesh,
            sigmaMap=self.physprops.wires.sigma,
            bc_type='Dirichlet',
            Solver=Solver
        )
        self._src = DC.Src.Dipole([], self.src_a, self.src_b)
        self._survey = DC.Survey([self._src])

        self._prob.pair(self._survey)
<|endoftext|>


<commit_before>from django.db import models
from django.db.models.signals import post_save, post_delete

from supertagging import settings
from supertagging.modules import process, clean_up

def save_handler(sender, **kwargs):
    if 'instance' in kwargs:
        process(kwargs['instance'])

def delete_handler(sender, **kwargs):
    if 'instance' in kwargs:
        clean_up(kwargs['instance'])

def setup():
    if not settings.AUTO_PROCESS:
        return

    try:
        for k,v in settings.MODULES.items():
            app_label, model_name = k.split('.')
            model = models.get_model(app_label, model_name)
            # Setup post save and post delete handlers
            post_save.connect(save_handler, sender=model)
            post_delete.connect(delete_handler, sender=model)

    except Exception, e:
        if settings.ST_DEBUG: raise Exception(e)
<commit_msg>Added queue support for the handlers<commit_after>from django.db import models
from django.db.models.signals import post_save, post_delete

from supertagging import settings
from supertagging.modules import process, clean_up, add_to_queue, remove_from_queue

def save_handler(sender, **kwargs):
    if 'instance' in kwargs:
        if settings.USE_QUEUE:
            add_to_queue(kwargs['instance'])
        else:
            process(kwargs['instance'])

def delete_handler(sender, **kwargs):
    if 'instance' in kwargs:
        if settings.USE_QUEUE:
            remove_from_queue(kwargs['instance'])
        else:
            clean_up(kwargs['instance'])

def setup():
    if not settings.AUTO_PROCESS:
        return

    try:
        for k,v in settings.MODULES.items():
            app_label, model_name = k.split('.')
            model = models.get_model(app_label, model_name)
            # Setup post save and post delete handlers
            post_save.connect(save_handler, sender=model)
            post_delete.connect(delete_handler, sender=model)
    except Exception, e:
        if settings.ST_DEBUG: raise Exception(e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals, absolute_import
import frappe
from frappe import _
import json
from frappe.model.document import Document
from frappe.core.doctype.user.user import extract_mentions
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\
	get_title, get_title_html
from frappe.utils import get_fullname
from frappe.website.render import clear_cache
from frappe.database.schema import add_column
from frappe.exceptions import ImplicitCommitError

class Comment(Document):
	def after_insert(self):
		self.notify_mentions()

		frappe.publish_realtime('new_communication', self.as_dict(),
			doctype=self.reference_doctype, docname=self.reference_name,
			after_commit=True)

	def validate(self):
		if not self.comment_email:
			self.comment_email = frappe.session.user
		self.content = frappe.utils.sanitize_html(self.content)

	def on_update(self):
		update_comment_in_doc(self)

	def on_trash(self):
		self.remove_comment_from_cache()
		frappe.publish_realtime('delete_communication', self.as_dict(),
			doctype= self.reference_doctype, docname = self.reference_name,
			after_commit=True)

	def remove_comment_from_cache(self):
		_comments = get_comments_from_parent(self)
		for c in _comments:
			if c.get("name")==self.name:
				_comments.remove(c)

		update_comments_in_parent(self.reference_doctype, self.reference_name, _comments)

	def notify_mentions(self):
		if self.reference_doctype and self.reference_name and self.content:
			mentions = extract_mentions(self.content)

			if not mentions:
				return

			sender_fullname = get_fullname(frappe.session.user)
			title = get_title(self.reference_doctype, self.reference_name)

			recipients = [frappe.db.get_value("User", {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, "email")
				for name in mentions]

			notification_message = _('''{0} mentioned you in a comment in {1} {2}''')\
				.format(frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title))

			notification_doc = {
				'type': 'Mention',
				'document_type': self.reference_doctype,
				'document_name': self.reference_name,
				'subject': notification_message,
				'from_user': frappe.session.user,
				'email_content': self.content
			}

			enqueue_create_notification(recipients, notification_doc)

def on_doctype_update():
	frappe.db.add_index("Comment", ["reference_doctype", "reference_name"])
	frappe.db.add_index("Comment", ["link_doctype", "link_name"])

def update_comment_in_doc(doc):
	"""Updates `_comments` (JSON) property in parent Document.
	Creates a column `_comments` if property does not exist.

	Only user created Communication or Comment of type Comment are saved.

	`_comments` format

		{
			"comment": [String],
			"by": [user],
			"name": [Comment Document name]
		}"""

	# only comments get updates, not likes, assignments etc.
	if doc.doctype == 'Comment' and doc.comment_type != 'Comment':
		return

	def get_truncated(content):
		return (content[:97] + '...') if len(content) > 100 else content

	if doc.reference_doctype and doc.reference_name and doc.content:
		_comments = get_comments_from_parent(doc)

		updated = False
		for c in _comments:
			if c.get("name")==doc.name:
				c["comment"] = get_truncated(doc.content)
				updated = True

		if not updated:
			_comments.append({
				"comment": get_truncated(doc.content),

				# "comment_email" for Comment and "sender" for Communication
				"by": getattr(doc, 'comment_email', None) or getattr(doc, 'sender', None) or doc.owner,
				"name": doc.name
			})

		update_comments_in_parent(doc.reference_doctype, doc.reference_name, _comments)

def get_comments_from_parent(doc):
	'''
	get the list of comments cached in the document record in the column
	`_comments`
	'''
	try:
		_comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]"

	except Exception as e:
		if frappe.db.is_missing_table_or_column(e):
			_comments = "[]"

		else:
			raise

	try:
		return json.loads(_comments)
	except ValueError:
		return []

def update_comments_in_parent(reference_doctype, reference_name, _comments):
	"""Updates `_comments` property in parent Document with given dict.

	:param _comments: Dict of comments."""
	if not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle"):
		return

	try:
		# use sql, so that we do not mess with the timestamp
		frappe.db.sql("""update `tab{0}` set `_comments`=%s where name=%s""".format(reference_doctype), # nosec
			(json.dumps(_comments[-50:]), reference_name))

	except Exception as e:
		if frappe.db.is_column_missing(e) and getattr(frappe.local, 'request', None):
			# missing column and in request, add column and update after commit
			frappe.local._comments = (getattr(frappe.local, "_comments", [])
				+ [(reference_doctype, reference_name, _comments)])

		elif frappe.db.is_data_too_long(e):
			raise frappe.DataTooLongException

		else:
			raise ImplicitCommitError

	else:
		if not frappe.flags.in_patch:
			reference_doc = frappe.get_doc(reference_doctype, reference_name)
			if getattr(reference_doc, "route", None):
				clear_cache(reference_doc.route)

def update_comments_in_parent_after_request():
	"""update _comments in parent if _comments column is missing"""
	if hasattr(frappe.local, "_comments"):
		for (reference_doctype, reference_name, _comments) in frappe.local._comments:
			add_column(reference_doctype, "_comments", "Text")
			update_comments_in_parent(reference_doctype, reference_name, _comments)

		frappe.db.commit()
<commit_msg>fix: list view comment count (#12156)<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals, absolute_import
import frappe
from frappe import _
import json
from frappe.model.document import Document
from frappe.core.doctype.user.user import extract_mentions
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\
	get_title, get_title_html
from frappe.utils import get_fullname
from frappe.website.render import clear_cache
from frappe.database.schema import add_column
from frappe.exceptions import ImplicitCommitError

class Comment(Document):
	def after_insert(self):
		self.notify_mentions()

		frappe.publish_realtime('new_communication', self.as_dict(),
			doctype=self.reference_doctype, docname=self.reference_name,
			after_commit=True)

	def validate(self):
		if not self.comment_email:
			self.comment_email = frappe.session.user
		self.content = frappe.utils.sanitize_html(self.content)

	def on_update(self):
		update_comment_in_doc(self)

	def on_trash(self):
		self.remove_comment_from_cache()
		frappe.publish_realtime('delete_communication', self.as_dict(),
			doctype= self.reference_doctype, docname = self.reference_name,
			after_commit=True)

	def remove_comment_from_cache(self):
		_comments = get_comments_from_parent(self)
		for c in _comments:
			if c.get("name")==self.name:
				_comments.remove(c)

		update_comments_in_parent(self.reference_doctype, self.reference_name, _comments)

	def notify_mentions(self):
		if self.reference_doctype and self.reference_name and self.content:
			mentions = extract_mentions(self.content)

			if not mentions:
				return

			sender_fullname = get_fullname(frappe.session.user)
			title = get_title(self.reference_doctype, self.reference_name)

			recipients = [frappe.db.get_value("User", {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, "email")
				for name in mentions]

			notification_message = _('''{0} mentioned you in a comment in {1} {2}''')\
				.format(frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title))

			notification_doc = {
				'type': 'Mention',
				'document_type': self.reference_doctype,
				'document_name': self.reference_name,
				'subject': notification_message,
				'from_user': frappe.session.user,
				'email_content': self.content
			}

			enqueue_create_notification(recipients, notification_doc)

def on_doctype_update():
	frappe.db.add_index("Comment", ["reference_doctype", "reference_name"])
	frappe.db.add_index("Comment", ["link_doctype", "link_name"])

def update_comment_in_doc(doc):
	"""Updates `_comments` (JSON) property in parent Document.
	Creates a column `_comments` if property does not exist.

	Only user created Communication or Comment of type Comment are saved.

	`_comments` format

		{
			"comment": [String],
			"by": [user],
			"name": [Comment Document name]
		}"""

	# only comments get updates, not likes, assignments etc.
	if doc.doctype == 'Comment' and doc.comment_type != 'Comment':
		return

	def get_truncated(content):
		return (content[:97] + '...') if len(content) > 100 else content

	if doc.reference_doctype and doc.reference_name and doc.content:
		_comments = get_comments_from_parent(doc)

		updated = False
		for c in _comments:
			if c.get("name")==doc.name:
				c["comment"] = get_truncated(doc.content)
				updated = True

		if not updated:
			_comments.append({
				"comment": get_truncated(doc.content),

				# "comment_email" for Comment and "sender" for Communication
				"by": getattr(doc, 'comment_email', None) or getattr(doc, 'sender', None) or doc.owner,
				"name": doc.name
			})

		update_comments_in_parent(doc.reference_doctype, doc.reference_name, _comments)

def get_comments_from_parent(doc):
	'''
	get the list of comments cached in the document record in the column
	`_comments`
	'''
	try:
		_comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]"

	except Exception as e:
		if frappe.db.is_missing_table_or_column(e):
			_comments = "[]"

		else:
			raise

	try:
		return json.loads(_comments)
	except ValueError:
		return []

def update_comments_in_parent(reference_doctype, reference_name, _comments):
	"""Updates `_comments` property in parent Document with given dict.

	:param _comments: Dict of comments."""
	if not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle"):
		return

	try:
		# use sql, so that we do not mess with the timestamp
		frappe.db.sql("""update `tab{0}` set `_comments`=%s where name=%s""".format(reference_doctype), # nosec
			(json.dumps(_comments[-100:]), reference_name))

	except Exception as e:
		if frappe.db.is_column_missing(e) and getattr(frappe.local, 'request', None):
			# missing column and in request, add column and update after commit
			frappe.local._comments = (getattr(frappe.local, "_comments", [])
				+ [(reference_doctype, reference_name, _comments)])

		elif frappe.db.is_data_too_long(e):
			raise frappe.DataTooLongException

		else:
			raise ImplicitCommitError

	else:
		if not frappe.flags.in_patch:
			reference_doc = frappe.get_doc(reference_doctype, reference_name)
			if getattr(reference_doc, "route", None):
				clear_cache(reference_doc.route)

def update_comments_in_parent_after_request():
	"""update _comments in parent if _comments column is missing"""
	if hasattr(frappe.local, "_comments"):
		for (reference_doctype, reference_name, _comments) in frappe.local._comments:
			add_column(reference_doctype, "_comments", "Text")
			update_comments_in_parent(reference_doctype, reference_name, _comments)

		frappe.db.commit()
<|endoftext|>


<commit_before>"""Entry point script; implements CLI."""

import argparse
import msvcrt
import os
import pickle
import sys

from src.pytorch_impl.src import classify as pyt_classify
from src.pytorch_impl.src import train as pyt_train
from src.tensorflow_impl import constants as tf_constants
from src.tensorflow_impl.src import classify as tf_classify
from src.tensorflow_impl.src import train as tf_train

def training_prompt():
    """
    Prompts the user with a warning message about overwriting the saved model.
    """
    print('WARNING: Training will overwrite the saved model (if it exists). EXECUTE Y/N?')
    while True:
        resp = msvcrt.getch().decode().lower()
        if resp == 'y':
            print('Loading...')
            return
        elif resp == 'n':
            sys.exit('Training aborted.')
        else:
            print('Press either the Y or N key.')

def print_prediction(prediction):
    """
    Prints the predicted label(s) by the model to the screen.

    Parameters:
        - prediction (str or dict)
            - Prediction(s) generated by model.
            - Type depends on whether a single image or a directory of images was classified.
    """
    if type(prediction) == str:
        print(prediction)
    elif type(prediction) == dict:
        keys, values = list(prediction.keys()), list(prediction.values())
        for key, value in zip(keys, values):
            print(key, value)

def parse_arguments():
    """
    Parses the CLI for specific arguments via the `argparse` library.

    Returns:
        - A dictionary mapping each argument to the given value.
        - Each string value is lowercased to prevent case sensitivity issues.
    """
    train_help = '(flag) Tells program to train the model.'
    resuming_help = ('(flag) Tells program to resume training off of a saved model '
                     'whose path is given by the --savepath arg.')
    train_dir_help = ('(str) Tells program where training dataset is; this directory should '
                      'contain subdirectories (representing each class) of images.')
    label_dict_path_help = ('(str) This path should point to a .pkl file of a dictionary mapping '
                            'strings (specifically, the names of the subdirectories in the '
                            '--train_dir arg) to numpy arrays. The shape of each label should '
                            'be the same and it should match the output shape of the model.')
    steps_help = '(int) Indicates how many images to train on (one gradient update per image).'
    savepath_help = ('(str) Path that indicates where the model will be saved and loaded from. '
                     'File extension should be: .pth')
    classify_help = ('(flag) Tells program to classify something using the saved '
                     'model from the --savepath arg.')
    source_help = '(str) A path to either an image or directory of images to classify.'
    implementation_help = ('(str) Either \'pytorch\' (default) or \'tensorflow\'. Note that '
                           'the TensorFlow implementation uses a `constants` module to hold '
                           'important paths instead of the CLI args for `train_dir`, '
                           '`label_dict_path`, and `savepath`. SETTING THESE ARGS FOR THE '
                           'TENSORFLOW IMPLEMENTATION IS POINTLESS!')
    save_paths_config_help = ('(flag) Values entered for --train_dir, --label_dict_path, and '
                              '--savepath will be saved to a dictionary mapping those names to '
                              'the values that were passed into them. This dictionary will be '
                              'saved into `paths.pkl`, so future entry of those CLI args will '
                              'no longer be necessary so long as the .pkl file exists.')
    parser = argparse.ArgumentParser()
    parser.add_argument('--train', action='store_true', help=train_help)
    parser.add_argument('--resuming', action='store_true', help=resuming_help)
    parser.add_argument('--train_dir', type=str, help=train_dir_help)
    parser.add_argument('--label_dict_path', type=str, help=label_dict_path_help)
    parser.add_argument('--steps', type=int, help=steps_help)
    parser.add_argument('--savepath', type=str, help=savepath_help)
    parser.add_argument('--classify', action='store_true', help=classify_help)
    parser.add_argument('--source', type=str, help=source_help)
    parser.add_argument('--implementation', type=str, help=implementation_help)
    parser.add_argument('--save_paths_config', action='store_true', help=save_paths_config_help)
    parser.set_defaults(resuming=False, implementation='pytorch')
    args = parser.parse_args()
    if args.save_paths_config:
        paths = {'train_dir': args.train_dir, 'label_dict_path': args.label_dict_path,
                 'savepath': args.savepath}
        with open('cats_vs_dogs/paths.pkl', 'wb') as f:
            pickle.dump(paths, f)
    if 'paths.pkl' in os.listdir('cats_vs_dogs'):
        with open('cats_vs_dogs/paths.pkl', 'rb') as f:
            paths = pickle.load(f)
        args.train_dir = paths['train_dir']
        args.label_dict_path = paths['label_dict_path']
        args.savepath = paths['savepath']
    args_dict = vars(args)
    args_dict = {key: value.lower() for key, value in args_dict.items() if type(value) == str}
    return vars(args)

def main(args):
    """
    Executes the program.

    Parameters:
        - args (dict)
            - A dictionary converted from an argparse.Namespace object.
    """
    if args['implementation'] == 'pytorch':
        if args['train']:
            training_prompt()
            pyt_train.main(args['train_dir'], args['label_dict_path'], args['steps'],
                           args['savepath'], resuming=args['resuming'])
        elif args['classify']:
            prediction = pyt_classify.main(args['source'], args['savepath'],
                                           args['label_dict_path'])
            print_prediction(prediction)
    elif args['implementation'] == 'tensorflow':
        if args['train']:
            training_prompt()
            tf_train.main(tf_constants.TRAIN_DIR, tf_constants.ENCODING, args['steps'],
                          tf_constants.SAVEPATH, tf_constants.TENSORBOARD_DIR,
                          resuming=args['resuming'])
        elif args['classify']:
            prediction = tf_classify.main(args['source'], tf_constants.SAVEPATH,
                                          tf_constants.ENCODING)
            print_prediction(prediction)

if __name__ == '__main__':
    args = parse_arguments()
    main(args)
<commit_msg>Dropping saved config support for now.<commit_after>"""Entry point script; implements CLI."""

import argparse
import msvcrt
import sys

from src.pytorch_impl.src import classify as pyt_classify
from src.pytorch_impl.src import train as pyt_train
from src.tensorflow_impl import constants as tf_constants
from src.tensorflow_impl.src import classify as tf_classify
from src.tensorflow_impl.src import train as tf_train

def training_prompt():
    """
    Prompts the user with a warning message about overwriting the saved model.
    """
    print('WARNING: Training will overwrite the saved model (if it exists). EXECUTE Y/N?')
    while True:
        resp = msvcrt.getch().decode().lower()
        if resp == 'y':
            print('Loading...')
            return
        elif resp == 'n':
            sys.exit('Training aborted.')
        else:
            print('Press either the Y or N key.')

def print_prediction(prediction):
    """
    Prints the predicted label(s) by the model to the screen.

    Parameters:
        - prediction (str or dict)
            - Prediction(s) generated by model.
            - Type depends on whether a single image or a directory of images was classified.
    """
    if type(prediction) == str:
        print(prediction)
    elif type(prediction) == dict:
        keys, values = list(prediction.keys()), list(prediction.values())
        for key, value in zip(keys, values):
            print(key, value)

def parse_arguments():
    """
    Parses the CLI for specific arguments via the `argparse` library.

    Returns:
        - A dictionary mapping each argument to the given value.
        - Each string value is lowercased to prevent case sensitivity issues.
    """
    train_help = '(flag) Tells program to train the model.'
    resuming_help = ('(flag) Tells program to resume training off of a saved model '
                     'whose path is given by the --savepath arg.')
    train_dir_help = ('(str) Tells program where training dataset is; this directory should '
                      'contain subdirectories (representing each class) of images.')
    label_dict_path_help = ('(str) This path should point to a .pkl file of a dictionary mapping '
                            'strings (specifically, the names of the subdirectories in the '
                            '--train_dir arg) to numpy arrays. The shape of each label should '
                            'be the same and it should match the output shape of the model.')
    savepath_help = ('(str) Path that indicates where the model will be saved and loaded from. '
                     'File extension should be: .pth')
    steps_help = '(int) Indicates how many images to train on (one gradient update per image).'
    classify_help = ('(flag) Tells program to classify something using the saved '
                     'model from the --savepath arg.')
    source_help = '(str) A path to either an image or directory of images to classify.'
    implementation_help = ('(str) Either \'pytorch\' (default) or \'tensorflow\'. Note that '
                           'the TensorFlow implementation uses a `constants` module to hold '
                           'important paths instead of the CLI args for `train_dir`, '
                           '`label_dict_path`, and `savepath`. SETTING THESE ARGS FOR THE '
                           'TENSORFLOW IMPLEMENTATION IS POINTLESS!')
    parser = argparse.ArgumentParser()
    parser.add_argument('--train', action='store_true', help=train_help)
    parser.add_argument('--resuming', action='store_true', help=resuming_help)
    parser.add_argument('--train_dir', type=str, help=train_dir_help)
    parser.add_argument('--label_dict_path', type=str, help=label_dict_path_help)
    parser.add_argument('--savepath', type=str, help=savepath_help)
    parser.add_argument('--steps', type=int, help=steps_help)
    parser.add_argument('--classify', action='store_true', help=classify_help)
    parser.add_argument('--source', type=str, help=source_help)
    parser.add_argument('--implementation', type=str, help=implementation_help)
    parser.set_defaults(resuming=False, implementation='pytorch')
    args = parser.parse_args()
    args_dict = vars(args)
    args_dict = {key: value.lower() for key, value in args_dict.items() if type(value) == str}
    return vars(args)

def main(args):
    """
    Executes the program.

    Parameters:
        - args (dict)
            - A dictionary converted from an argparse.Namespace object.
    """
    if args['implementation'] == 'pytorch':
        if args['train']:
            training_prompt()
            pyt_train.main(args['train_dir'], args['label_dict_path'], args['steps'],
                           args['savepath'], resuming=args['resuming'])
        elif args['classify']:
            prediction = pyt_classify.main(args['source'], args['savepath'],
                                           args['label_dict_path'])
            print_prediction(prediction)
    elif args['implementation'] == 'tensorflow':
        if args['train']:
            training_prompt()
            tf_train.main(tf_constants.TRAIN_DIR, tf_constants.ENCODING, args['steps'],
                          tf_constants.SAVEPATH, tf_constants.TENSORBOARD_DIR,
                          resuming=args['resuming'])
        elif args['classify']:
            prediction = tf_classify.main(args['source'], tf_constants.SAVEPATH,
                                          tf_constants.ENCODING)
            print_prediction(prediction)

if __name__ == '__main__':
    args = parse_arguments()
    main(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import vim
import re
import os
import os.path
import subprocess
import tempfile
from leaderf.utils import *
from leaderf.explorer import *
from leaderf.manager import *

#*****************************************************
# BufTagExplorer
#*****************************************************
class BufTagExplorer(Explorer):
    def __init__(self):
        self._ctags = lfEval("g:Lf_Ctags")
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))
        self._tag_list = {}        # a dict with (key, value) = (buffer number, taglist)
        self._buf_changedtick = {} # a dict with (key, value) = (buffer number, changedtick)
        for buf in vim.buffers:
            if buf.options["buflisted"]:
                changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buf.number))
                self._buf_changedtick[buf.number] = changedtick

    def getContent(self, *args, **kwargs):
        tag_list = []
        if len(args) > 0: # all buffers
            cur_buffer = vim.current.buffer
            for b in vim.buffers:
                if b.options["buflisted"]:
                    if lfEval("bufloaded(%d)" % b.number) == '0':
                        saved_eventignore = vim.options["eventignore"]
                        vim.options["eventignore"] = "all"
                        try:
                            lfCmd("silent hide buffer %d" % b.number)
                        finally:
                            vim.options["eventignore"] = saved_eventignore
                    tag_list.extend(self._getTaglist(b))
            if vim.current.buffer is not cur_buffer:
                vim.current.buffer = cur_buffer
        else:
            tag_list = self._getTaglist(vim.current.buffer)
        return tag_list

    def _getTaglist(self, buffer):
        changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buffer.number))
        # there is no change since last call
        if changedtick == self._buf_changedtick.get(buffer.number, -1):
            if buffer.number in self._tag_list:
                return self._tag_list[buffer.number]
        else:
            self._buf_changedtick[buffer.number] = changedtick

        if buffer.options["filetype"] == b"cpp":
            extra_options = "--c++-kinds=+p"
        elif buffer.options["filetype"] == b"c":
            extra_options = "--c-kinds=+p"
        else:
            extra_options = ""

        # {tagname}<Tab>{tagfile}<Tab>{tagaddress}[;"<Tab>{tagfield}..]
        # {tagname}<Tab>{tagfile}<Tab>{tagaddress};"<Tab>{kind}<Tab>{scope}
        process = subprocess.Popen("{} -n -u --fields=Ks {} -f- -L- ".format(self._ctags,
                                                                             extra_options),
                                   shell=True,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   universal_newlines=True)
        if buffer.options["modified"] == True:
            with tempfile.NamedTemporaryFile(mode='w+',
                                             suffix='_'+os.path.basename(buffer.name),
                                             delete=False) as f:
                for line in buffer[:]:
                    f.write(line + '\n')
                file_name = f.name
            out = process.communicate(lfDecode(file_name))
            os.remove(file_name)
        else:
            out = process.communicate(lfDecode(buffer.name))

        if out[1]:
            lfCmd("echoerr '%s'" % escQuote(out[1].rstrip()))

        if not out[0]:
            return []

        # a list of [tag, file, line, kind, scope]
        output = [line.split('\t') for line in out[0].splitlines()]
        if len(output[0]) < 4:
            lfCmd("echoerr '%s'" % escQuote(out[0].rstrip()))
            return []

        tag_total_len = 0
        max_kind_len = 0
        for _, item  in enumerate(output):
            tag_total_len += len(item[0])
            kind_len = len(item[3])
            if kind_len > max_kind_len:
                max_kind_len = kind_len
        ave_taglen = tag_total_len // len(output)
        tag_len = ave_taglen * 3 // 2

        tab_len = buffer.options["shiftwidth"]
        std_tag_kind_len = tag_len // tab_len * tab_len + tab_len + max_kind_len

        tag_list = []
        for _, item  in enumerate(output):
            scope = item[4] if len(item) > 4 else "Global"
            tag_kind = "{:{taglen}s}\t{}".format(item[0],   # tag
                                                 item[3],   # kind
                                                 taglen=tag_len
                                                 )
            tag_kind_len = int(lfEval("strdisplaywidth('%s')" % escQuote(tag_kind)))
            num = std_tag_kind_len - tag_kind_len
            space_num = num if num > 0 else 0
            bufname = buffer.name if vim.options["autochdir"] else lfRelpath(buffer.name)
            line = "{}{}\t{}\t{:2s}{}\t{}".format(tag_kind,
                                                  ' ' * space_num,
                                                  scope,          # scope
                                                  ' ',
                                                  bufname,        # file
                                                  item[2][:-2]    # line
                                                  )
            tag_list.append(line)
            if self._supports_preview:
                # code = "{:{taglen}s}\t{}".format(' ' * len(item[0]),
                #                                  buffer[int(item[2][:-2]) - 1].lstrip(),
                #                                  taglen=tag_len
                #                                  )
                code = "\t\t{}".format(buffer[int(item[2][:-2]) - 1].lstrip())
                tag_list.append(code)

        self._tag_list[buffer.number] = tag_list

        return tag_list

    def getStlCategory(self):
        return 'BufTag'

    def getStlCurDir(self):
        return escQuote(lfEncode(os.getcwd()))

    def isFilePath(self):
        return False

    def removeCache(self, buf_number):
        if buf_number in self._tag_list:
            del self._tag_list[buf_number]

        if buf_number in self._buf_changedtick:
            del self._buf_changedtick[buf_number]

#*****************************************************
# BufTagExplManager
#*****************************************************
class BufTagExplManager(Manager):
    def __init__(self):
        super(BufTagExplManager, self).__init__()
        self._match_ids = []
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))

    def _getExplClass(self):
        return BufTagExplorer

    def _defineMaps(self):
        lfCmd("call leaderf#bufTagExplMaps()")

    def _acceptSelection(self, *args, **kwargs):
        if len(args) == 0:
            return
        line = args[0]
        if line[0].isspace():
            buffer = args[1]
            line_nr = args[2]
            line = buffer[line_nr - 2]
        # {tag} {kind} {scope} {file} {line}
        items = re.split(" *\t *", line)
        tagname = items[0]
        tagfile, line_nr = items[3:]
        lfCmd("hide buffer +%s %s" % (line_nr, escSpecial(tagfile)))
        lfCmd("norm! ^")
        lfCmd("call search('\V%s', 'Wc', line('.'))" % escQuote(tagname))
        lfCmd("norm! zz")

    def _getDigest(self, line, mode):
        """
        specify what part in the line to be processed and highlighted
        Args:
            mode: 0, return the whole line
                  1, return the tagname
                  2, return the remaining part
        """
        if mode == 0:
            return line
        elif mode == 1:
            return re.split(" *\t *", line, 1)[0]
        else:
            return re.split(" *\t *", line, 1)[1]

    def _getDigestStartPos(self, line, mode):
        """
        return the start position of the digest returned by _getDigest()
        Args:
            mode: 0, return the start position of the whole line
                  1, return the start position of tagname
                  2, return the start position remaining part
        """
        if mode == 0:
            return 0
        elif mode == 1:
            return 0
        else:
            return len(line) - len(re.split(" *\t *", line, 1)[1])

    def _createHelp(self):
        help = []
        help.append('" <CR>/<double-click>/o : open file under cursor')
        help.append('" x : open file under cursor in a horizontally split window')
        help.append('" v : open file under cursor in a vertically split window')
        help.append('" t : open file under cursor in a new tabpage')
        help.append('" i : switch to input mode')
        help.append('" q : quit')
        help.append('" <F1> : toggle this help')
        help.append('" ---------------------------------------------------------')
        return help

    def _afterEnter(self):
        super(BufTagExplManager, self)._afterEnter()
        id = int(lfEval('''matchadd('Lf_hl_buftagKind', '^[^\t]*\t\zs\S\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagScopeType', '[^\t]*\t\S\+\s*\zs\w\+:')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagScope', '^[^\t]*\t\S\+\s*\(\w\+:\)\=\zs\S\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagDirname', '[^\t]*\t\S\+\s*\S\+\s*\zs[^\t]\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagLineNum', '\d\+$')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagCode', '^\s\+.*')'''))
        self._match_ids.append(id)

    def _beforeExit(self):
        super(BufTagExplManager, self)._beforeExit()
        for i in self._match_ids:
            lfCmd("silent! call matchdelete(%d)" % i)
        self._match_ids = []

    def _getUnit(self):
        """
        indicates how many lines are considered as a unit
        """
        if self._supports_preview:
            return 2
        else:
            return 1

    def _supportsRefine(self):
        return True

    def _fuzzyFilter(self, is_full_path, get_weight, iterable):
        """
        return a list, each item is a triple (weight, line1, line2)
        """
        if self._supports_preview:
            if len(iterable) < 2:
                return []
            getDigest = partial(self._getDigest, mode=0 if is_full_path else 1)
            triples = ((get_weight(getDigest(line)), line, iterable[2*i+1])
                       for i, line in enumerate(iterable[::2]))
            return (t for t in triples if t[0])
        else:
            return super(BufTagExplManager, self)._fuzzyFilter(is_full_path,
                                                               get_weight,
                                                               iterable)

    def _refineFilter(self, first_get_weight, get_weight, iterable):
        if self._supports_preview:
            if len(iterable) < 2:
                return []
            getDigest = self._getDigest
            tuples = ((first_get_weight(getDigest(line, 1)), get_weight(getDigest(line, 2)),
                       line, iterable[2*i+1]) for i, line in enumerate(iterable[::2]))
            return ((i[0] + i[1], i[2], i[3]) for i in tuples if i[0] and i[1])
        else:
            return super(BufTagExplManager, self)._refineFilter(first_get_weight,
                                                                get_weight,
                                                                iterable)

    def _regexFilter(self, iterable):
        try:
            if ('-2' == lfEval("g:LfNoErrMsgMatch('', '%s')" % escQuote(self._cli.pattern))):
                return iter([])
            else:
                result = []
                for i, line in enumerate(iterable[::2]):
                    if ('-1' != lfEval("g:LfNoErrMsgMatch('%s', '%s')" %
                        (escQuote(self._getDigest(line, 1).strip()),
                            escQuote(self._cli.pattern)))):
                        result.append(line)
                        result.append(iterable[2*i+1])
                return result
        except vim.error:
            return iter([])

    def _getList(self, pairs):
        """
        return a list constructed from pairs
        Args:
            pairs: a list of tuple(weight, line, ...)
        """
        if self._supports_preview:
            result = []
            for _, p in enumerate(pairs):
                result.append(p[1])
                result.append(p[2])
            return result
        else:
            return super(BufTagExplManager, self)._getList(pairs)

    def _toUp(self):
        if self._supports_preview:
            lfCmd("norm! 2k")
        else:
            super(BufTagExplManager, self)._toUp()

    def _toDown(self):
        if self._supports_preview:
            lfCmd("norm! 3jk")
        else:
            super(BufTagExplManager, self)._toDown()

    def removeCache(self, buf_number):
        self._getExplorer().removeCache(buf_number)

#*****************************************************
# bufTagExplManager is a singleton
#*****************************************************
bufTagExplManager = BufTagExplManager()

__all__ = ['bufTagExplManager']
<commit_msg>don't ignore event<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import vim
import re
import os
import os.path
import subprocess
import tempfile
from leaderf.utils import *
from leaderf.explorer import *
from leaderf.manager import *

#*****************************************************
# BufTagExplorer
#*****************************************************
class BufTagExplorer(Explorer):
    def __init__(self):
        self._ctags = lfEval("g:Lf_Ctags")
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))
        self._tag_list = {}        # a dict with (key, value) = (buffer number, taglist)
        self._buf_changedtick = {} # a dict with (key, value) = (buffer number, changedtick)
        for buf in vim.buffers:
            if buf.options["buflisted"]:
                changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buf.number))
                self._buf_changedtick[buf.number] = changedtick

    def getContent(self, *args, **kwargs):
        tag_list = []
        if len(args) > 0: # all buffers
            cur_buffer = vim.current.buffer
            for b in vim.buffers:
                if b.options["buflisted"]:
                    if lfEval("bufloaded(%d)" % b.number) == '0':
                        lfCmd("silent hide buffer %d" % b.number)
                    tag_list.extend(self._getTaglist(b))
            if vim.current.buffer is not cur_buffer:
                vim.current.buffer = cur_buffer
        else:
            tag_list = self._getTaglist(vim.current.buffer)
        return tag_list

    def _getTaglist(self, buffer):
        changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buffer.number))
        # there is no change since last call
        if changedtick == self._buf_changedtick.get(buffer.number, -1):
            if buffer.number in self._tag_list:
                return self._tag_list[buffer.number]
        else:
            self._buf_changedtick[buffer.number] = changedtick

        if buffer.options["filetype"] == b"cpp":
            extra_options = "--c++-kinds=+p"
        elif buffer.options["filetype"] == b"c":
            extra_options = "--c-kinds=+p"
        else:
            extra_options = ""

        # {tagname}<Tab>{tagfile}<Tab>{tagaddress}[;"<Tab>{tagfield}..]
        # {tagname}<Tab>{tagfile}<Tab>{tagaddress};"<Tab>{kind}<Tab>{scope}
        process = subprocess.Popen("{} -n -u --fields=Ks {} -f- -L- ".format(self._ctags,
                                                                             extra_options),
                                   shell=True,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   universal_newlines=True)
        if buffer.options["modified"] == True:
            with tempfile.NamedTemporaryFile(mode='w+',
                                             suffix='_'+os.path.basename(buffer.name),
                                             delete=False) as f:
                for line in buffer[:]:
                    f.write(line + '\n')
                file_name = f.name
            out = process.communicate(lfDecode(file_name))
            os.remove(file_name)
        else:
            out = process.communicate(lfDecode(buffer.name))

        if out[1]:
            lfCmd("echoerr '%s'" % escQuote(out[1].rstrip()))

        if not out[0]:
            return []

        # a list of [tag, file, line, kind, scope]
        output = [line.split('\t') for line in out[0].splitlines()]
        if len(output[0]) < 4:
            lfCmd("echoerr '%s'" % escQuote(out[0].rstrip()))
            return []

        tag_total_len = 0
        max_kind_len = 0
        for _, item  in enumerate(output):
            tag_total_len += len(item[0])
            kind_len = len(item[3])
            if kind_len > max_kind_len:
                max_kind_len = kind_len
        ave_taglen = tag_total_len // len(output)
        tag_len = ave_taglen * 3 // 2

        tab_len = buffer.options["shiftwidth"]
        std_tag_kind_len = tag_len // tab_len * tab_len + tab_len + max_kind_len

        tag_list = []
        for _, item  in enumerate(output):
            scope = item[4] if len(item) > 4 else "Global"
            tag_kind = "{:{taglen}s}\t{}".format(item[0],   # tag
                                                 item[3],   # kind
                                                 taglen=tag_len
                                                 )
            tag_kind_len = int(lfEval("strdisplaywidth('%s')" % escQuote(tag_kind)))
            num = std_tag_kind_len - tag_kind_len
            space_num = num if num > 0 else 0
            bufname = buffer.name if vim.options["autochdir"] else lfRelpath(buffer.name)
            line = "{}{}\t{}\t{:2s}{}\t{}".format(tag_kind,
                                                  ' ' * space_num,
                                                  scope,          # scope
                                                  ' ',
                                                  bufname,        # file
                                                  item[2][:-2]    # line
                                                  )
            tag_list.append(line)
            if self._supports_preview:
                # code = "{:{taglen}s}\t{}".format(' ' * len(item[0]),
                #                                  buffer[int(item[2][:-2]) - 1].lstrip(),
                #                                  taglen=tag_len
                #                                  )
                code = "\t\t{}".format(buffer[int(item[2][:-2]) - 1].lstrip())
                tag_list.append(code)

        self._tag_list[buffer.number] = tag_list

        return tag_list

    def getStlCategory(self):
        return 'BufTag'

    def getStlCurDir(self):
        return escQuote(lfEncode(os.getcwd()))

    def isFilePath(self):
        return False

    def removeCache(self, buf_number):
        if buf_number in self._tag_list:
            del self._tag_list[buf_number]

        if buf_number in self._buf_changedtick:
            del self._buf_changedtick[buf_number]

#*****************************************************
# BufTagExplManager
#*****************************************************
class BufTagExplManager(Manager):
    def __init__(self):
        super(BufTagExplManager, self).__init__()
        self._match_ids = []
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))

    def _getExplClass(self):
        return BufTagExplorer

    def _defineMaps(self):
        lfCmd("call leaderf#bufTagExplMaps()")

    def _acceptSelection(self, *args, **kwargs):
        if len(args) == 0:
            return
        line = args[0]
        if line[0].isspace():
            buffer = args[1]
            line_nr = args[2]
            line = buffer[line_nr - 2]
        # {tag} {kind} {scope} {file} {line}
        items = re.split(" *\t *", line)
        tagname = items[0]
        tagfile, line_nr = items[3:]
        lfCmd("hide buffer +%s %s" % (line_nr, escSpecial(tagfile)))
        lfCmd("norm! ^")
        lfCmd("call search('\V%s', 'Wc', line('.'))" % escQuote(tagname))
        lfCmd("norm! zz")

    def _getDigest(self, line, mode):
        """
        specify what part in the line to be processed and highlighted
        Args:
            mode: 0, return the whole line
                  1, return the tagname
                  2, return the remaining part
        """
        if mode == 0:
            return line
        elif mode == 1:
            return re.split(" *\t *", line, 1)[0]
        else:
            return re.split(" *\t *", line, 1)[1]

    def _getDigestStartPos(self, line, mode):
        """
        return the start position of the digest returned by _getDigest()
        Args:
            mode: 0, return the start position of the whole line
                  1, return the start position of tagname
                  2, return the start position remaining part
        """
        if mode == 0:
            return 0
        elif mode == 1:
            return 0
        else:
            return len(line) - len(re.split(" *\t *", line, 1)[1])

    def _createHelp(self):
        help = []
        help.append('" <CR>/<double-click>/o : open file under cursor')
        help.append('" x : open file under cursor in a horizontally split window')
        help.append('" v : open file under cursor in a vertically split window')
        help.append('" t : open file under cursor in a new tabpage')
        help.append('" i : switch to input mode')
        help.append('" q : quit')
        help.append('" <F1> : toggle this help')
        help.append('" ---------------------------------------------------------')
        return help

    def _afterEnter(self):
        super(BufTagExplManager, self)._afterEnter()
        id = int(lfEval('''matchadd('Lf_hl_buftagKind', '^[^\t]*\t\zs\S\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagScopeType', '[^\t]*\t\S\+\s*\zs\w\+:')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagScope', '^[^\t]*\t\S\+\s*\(\w\+:\)\=\zs\S\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagDirname', '[^\t]*\t\S\+\s*\S\+\s*\zs[^\t]\+')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagLineNum', '\d\+$')'''))
        self._match_ids.append(id)
        id = int(lfEval('''matchadd('Lf_hl_buftagCode', '^\s\+.*')'''))
        self._match_ids.append(id)

    def _beforeExit(self):
        super(BufTagExplManager, self)._beforeExit()
        for i in self._match_ids:
            lfCmd("silent! call matchdelete(%d)" % i)
        self._match_ids = []

    def _getUnit(self):
        """
        indicates how many lines are considered as a unit
        """
        if self._supports_preview:
            return 2
        else:
            return 1

    def _supportsRefine(self):
        return True

    def _fuzzyFilter(self, is_full_path, get_weight, iterable):
        """
        return a list, each item is a triple (weight, line1, line2)
        """
        if self._supports_preview:
            if len(iterable) < 2:
                return []
            getDigest = partial(self._getDigest, mode=0 if is_full_path else 1)
            triples = ((get_weight(getDigest(line)), line, iterable[2*i+1])
                       for i, line in enumerate(iterable[::2]))
            return (t for t in triples if t[0])
        else:
            return super(BufTagExplManager, self)._fuzzyFilter(is_full_path,
                                                               get_weight,
                                                               iterable)

    def _refineFilter(self, first_get_weight, get_weight, iterable):
        if self._supports_preview:
            if len(iterable) < 2:
                return []
            getDigest = self._getDigest
            tuples = ((first_get_weight(getDigest(line, 1)), get_weight(getDigest(line, 2)),
                       line, iterable[2*i+1]) for i, line in enumerate(iterable[::2]))
            return ((i[0] + i[1], i[2], i[3]) for i in tuples if i[0] and i[1])
        else:
            return super(BufTagExplManager, self)._refineFilter(first_get_weight,
                                                                get_weight,
                                                                iterable)

    def _regexFilter(self, iterable):
        try:
            if ('-2' == lfEval("g:LfNoErrMsgMatch('', '%s')" % escQuote(self._cli.pattern))):
                return iter([])
            else:
                result = []
                for i, line in enumerate(iterable[::2]):
                    if ('-1' != lfEval("g:LfNoErrMsgMatch('%s', '%s')" %
                        (escQuote(self._getDigest(line, 1).strip()),
                            escQuote(self._cli.pattern)))):
                        result.append(line)
                        result.append(iterable[2*i+1])
                return result
        except vim.error:
            return iter([])

    def _getList(self, pairs):
        """
        return a list constructed from pairs
        Args:
            pairs: a list of tuple(weight, line, ...)
        """
        if self._supports_preview:
            result = []
            for _, p in enumerate(pairs):
                result.append(p[1])
                result.append(p[2])
            return result
        else:
            return super(BufTagExplManager, self)._getList(pairs)

    def _toUp(self):
        if self._supports_preview:
            lfCmd("norm! 2k")
        else:
            super(BufTagExplManager, self)._toUp()

    def _toDown(self):
        if self._supports_preview:
            lfCmd("norm! 3jk")
        else:
            super(BufTagExplManager, self)._toDown()

    def removeCache(self, buf_number):
        self._getExplorer().removeCache(buf_number)

#*****************************************************
# bufTagExplManager is a singleton
#*****************************************************
bufTagExplManager = BufTagExplManager()

__all__ = ['bufTagExplManager']
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""celerybeat

.. program:: celerybeat

.. cmdoption:: -s, --schedule

    Path to the schedule database. Defaults to ``celerybeat-schedule``.
    The extension ".db" will be appended to the filename.

.. cmdoption:: -f, --logfile

    Path to log file. If no logfile is specified, ``stderr`` is used.

.. cmdoption:: -l, --loglevel

    Logging level, choose between ``DEBUG``, ``INFO``, ``WARNING``,
    ``ERROR``, ``CRITICAL``, or ``FATAL``.

"""
import sys
import optparse
import traceback

import celery
from celery import conf
from celery import platform
from celery.log import emergency_error
from celery.beat import ClockService
from celery.utils import info

STARTUP_INFO_FMT = """
Configuration ->
    . broker -> %(conninfo)s
    . schedule -> %(schedule)s
    . logfile -> %(logfile)s@%(loglevel)s
""".strip()

OPTION_LIST = (
    optparse.make_option('-s', '--schedule',
            default=conf.CELERYBEAT_SCHEDULE_FILENAME,
            action="store", dest="schedule",
            help="Path to the schedule database. The extension \
                    '.db' will be appended to the filename. Default: %s" % (
                    conf.CELERYBEAT_SCHEDULE_FILENAME)),
    optparse.make_option('-f', '--logfile', default=conf.CELERYBEAT_LOG_FILE,
            action="store", dest="logfile",
            help="Path to log file."),
    optparse.make_option('-l', '--loglevel',
            default=conf.CELERYBEAT_LOG_LEVEL,
            action="store", dest="loglevel",
            help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL/FATAL."),
)

def run_clockservice(loglevel=conf.CELERYBEAT_LOG_LEVEL,
        logfile=conf.CELERYBEAT_LOG_FILE,
        schedule=conf.CELERYBEAT_SCHEDULE_FILENAME, **kwargs):
    """Starts the celerybeat clock server."""

    print("celerybeat %s is starting." % celery.__version__)

    # Setup logging
    if not isinstance(loglevel, int):
        loglevel = conf.LOG_LEVELS[loglevel.upper()]

    # Run the worker init handler.
    # (Usually imports task modules and such.)
    from celery.loaders import current_loader
    current_loader().init_worker()

    # Dump configuration to screen so we have some basic information
    # when users sends e-mails.

    print(STARTUP_INFO_FMT % {
            "conninfo": info.format_broker_info(),
            "logfile": logfile or "@stderr",
            "loglevel": conf.LOG_LEVELS[loglevel],
            "schedule": schedule,
    })

    print("celerybeat has started.")
    arg_start = "manage" in sys.argv[0] and 2 or 1
    platform.set_process_title("celerybeat",
                               info=" ".join(sys.argv[arg_start:]))

    def _run_clock():
        from celery.log import setup_logger
        logger = setup_logger(loglevel, logfile)
        clockservice = ClockService(logger=logger, schedule_filename=schedule)

        try:
            clockservice.start()
        except Exception, e:
            emergency_error(logfile,
                    "celerybeat raised exception %s: %s\n%s" % (
                            e.__class__, e, traceback.format_exc()))

    _run_clock()

def parse_options(arguments):
    """Parse the available options to ``celeryd``."""
    parser = optparse.OptionParser(option_list=OPTION_LIST)
    options, values = parser.parse_args(arguments)
    return options

def main():
    options = parse_options(sys.argv[1:])
    run_clockservice(**vars(options))

if __name__ == "__main__":
    main()
<commit_msg>celerybeat: Install SIGTERM+SIGINT handler that syncs the schedule to disk at shutdown.<commit_after>#!/usr/bin/env python
"""celerybeat

.. program:: celerybeat

.. cmdoption:: -s, --schedule

    Path to the schedule database. Defaults to ``celerybeat-schedule``.
    The extension ".db" will be appended to the filename.

.. cmdoption:: -f, --logfile

    Path to log file. If no logfile is specified, ``stderr`` is used.

.. cmdoption:: -l, --loglevel

    Logging level, choose between ``DEBUG``, ``INFO``, ``WARNING``,
    ``ERROR``, ``CRITICAL``, or ``FATAL``.

"""
import sys
import optparse
import traceback

import celery
from celery import conf
from celery import platform
from celery.log import emergency_error
from celery.beat import ClockService
from celery.utils import info

STARTUP_INFO_FMT = """
Configuration ->
    . broker -> %(conninfo)s
    . schedule -> %(schedule)s
    . logfile -> %(logfile)s@%(loglevel)s
""".strip()

OPTION_LIST = (
    optparse.make_option('-s', '--schedule',
            default=conf.CELERYBEAT_SCHEDULE_FILENAME,
            action="store", dest="schedule",
            help="Path to the schedule database. The extension \
                    '.db' will be appended to the filename. Default: %s" % (
                    conf.CELERYBEAT_SCHEDULE_FILENAME)),
    optparse.make_option('-f', '--logfile', default=conf.CELERYBEAT_LOG_FILE,
            action="store", dest="logfile",
            help="Path to log file."),
    optparse.make_option('-l', '--loglevel',
            default=conf.CELERYBEAT_LOG_LEVEL,
            action="store", dest="loglevel",
            help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL/FATAL."),
)

def run_clockservice(loglevel=conf.CELERYBEAT_LOG_LEVEL,
        logfile=conf.CELERYBEAT_LOG_FILE,
        schedule=conf.CELERYBEAT_SCHEDULE_FILENAME, **kwargs):
    """Starts the celerybeat clock server."""

    print("celerybeat %s is starting." % celery.__version__)

    # Setup logging
    if not isinstance(loglevel, int):
        loglevel = conf.LOG_LEVELS[loglevel.upper()]

    # Run the worker init handler.
    # (Usually imports task modules and such.)
    from celery.loaders import current_loader
    current_loader().init_worker()

    # Dump configuration to screen so we have some basic information
    # when users sends e-mails.

    print(STARTUP_INFO_FMT % {
            "conninfo": info.format_broker_info(),
            "logfile": logfile or "@stderr",
            "loglevel": conf.LOG_LEVELS[loglevel],
            "schedule": schedule,
    })

    print("celerybeat has started.")
    arg_start = "manage" in sys.argv[0] and 2 or 1
    platform.set_process_title("celerybeat",
                               info=" ".join(sys.argv[arg_start:]))

    def _run_clock():
        from celery.log import setup_logger
        logger = setup_logger(loglevel, logfile)
        clockservice = ClockService(logger=logger, schedule_filename=schedule)

        try:
            install_sync_handler(clockservice)
            clockservice.start()
        except Exception, e:
            emergency_error(logfile,
                    "celerybeat raised exception %s: %s\n%s" % (
                            e.__class__, e, traceback.format_exc()))

    _run_clock()

def install_sync_handler(beat):
    """Install a ``SIGTERM`` + ``SIGINT`` handler that saves
    the celerybeat schedule."""

    def _sync(signum, frame):
        beat.sync()
        raise SystemExit()

    platform.install_signal_handler("SIGTERM", _sync)
    platform.install_signal_handler("SIGINT", _sync)

def parse_options(arguments):
    """Parse the available options to ``celeryd``."""
    parser = optparse.OptionParser(option_list=OPTION_LIST)
    options, values = parser.parse_args(arguments)
    return options

def main():
    options = parse_options(sys.argv[1:])
    run_clockservice(**vars(options))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
import phonenumbers

class FindSpam:
    rules = [
        {'regex': u"(?i)\\b(baba(ji)?|fifa.*coins?|nike|tosterone|bajotz|vashi?k[ae]r[ae]n|sumer|kolcak|porn|molvi|judi bola|ituBola.com|lost lover|11s|acai|skin care|rejuvenated skin|LifeForce|swtor2credits|me2.do|black magic|bam2u|Neuro(3X|flexyn)|Nutra|TesteroneXL|Bowtrol|Slim ?Genix|Cleanse EFX|Babyliss ?Pro|Forskolin|Blackline Elite|TestCore Pro|Xtreme Antler|Maxx Test 3000|Cheap Wigs?|jivam|(Improve )?Brain Power|aging skin|acne( prone)? skin|(skin )?eye serum|skin (serum|eye)|(fake|original) (passports?|driver'?s? licen[cs]e|ID cards?)|bagprada|6611165613)\\b|ಌ|(support|service|helpline)( phone)? number|1[ -]?866[ -]?978[ -]?6819|(>>>>|===>)(?s).*http", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True},
        {'regex': u"(?i)\\b(fifabay)\\b", 'all': True, 'reason': "Bad keyword in {}", 'sites': [], 'title': True, 'body': True, 'username': True},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles? build(ing)?|muscles?( (grow(th)?|diets?))?|anti aging|SkinCentric|loo?s[es] weight|wrinkles?)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,10}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False},
        {'regex': u"^(?=.*[A-Z])[^a-z]*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'username': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False},
        {'regex': u"(?i)(online ?kelas|wowtoes|ipubsoft|orabank|powerigfaustralia|cfpchampionship2015playofflive|optimalstackfacts|maletestosteronebooster|x4facts|tripleeffectseyeserum|healthcaresup|garciniacambogiaprofacts|filerepairforum|lxwpro-t|casque-beatsbydre|tenderpublish|elliskinantiaging|funmac|lovebiscuits|Eglobalfitness)", 'sites': [], 'all': True, 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True},
        {'regex': u"([a-zA-Z])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer):
        result = []
        body = re.sub("
.*?
", "", body, flags=re.DOTALL)
        body = re.sub(".*?", "", body, flags=re.DOTALL)
        for rule in FindSpam.rules:
            if rule['all'] != (site in rule['sites']):
                matched_title = re.compile(rule['regex'], re.UNICODE).findall(title)
                matched_username = re.compile(rule['regex'], re.UNICODE).findall(user_name)
                matched_body = re.compile(rule['regex'], re.UNICODE).findall(body)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body']:
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<commit_msg>+hotmail support --autopull<commit_after># -*- coding: utf-8 -*-
import re
import phonenumbers

class FindSpam:
    rules = [
        {'regex': u"(?i)\\b(baba(ji)?|fifa.*coins?|hotmail (password)? support number|nike|tosterone|bajotz|vashi?k[ae]r[ae]n|sumer|kolcak|porn|molvi|judi bola|ituBola.com|lost lover|11s|acai|skin care|rejuvenated skin|LifeForce|swtor2credits|me2.do|black magic|bam2u|Neuro(3X|flexyn)|Nutra|TesteroneXL|Bowtrol|Slim ?Genix|Cleanse EFX|Babyliss ?Pro|Forskolin|Blackline Elite|TestCore Pro|Xtreme Antler|Maxx Test 3000|Cheap Wigs?|jivam|(Improve )?Brain Power|aging skin|acne( prone)? skin|(skin )?eye serum|skin (serum|eye)|(fake|original) (passports?|driver'?s? licen[cs]e|ID cards?)|bagprada|6611165613)\\b|ಌ|(support|service|helpline)( phone)? number|1[ -]?866[ -]?978[ -]?6819|(>>>>|===>)(?s).*http", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True},
        {'regex': u"(?i)\\b(fifabay)\\b", 'all': True, 'reason': "Bad keyword in {}", 'sites': [], 'title': True, 'body': True, 'username': True},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles? build(ing)?|muscles?( (grow(th)?|diets?))?|anti aging|SkinCentric|loo?s[es] weight|wrinkles?)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,10}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False},
        {'regex': u"^(?=.*[A-Z])[^a-z]*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'username': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False},
        {'regex': u"(?i)(online ?kelas|wowtoes|ipubsoft|orabank|powerigfaustralia|cfpchampionship2015playofflive|optimalstackfacts|maletestosteronebooster|x4facts|tripleeffectseyeserum|healthcaresup|garciniacambogiaprofacts|filerepairforum|lxwpro-t|casque-beatsbydre|tenderpublish|elliskinantiaging|funmac|lovebiscuits|Eglobalfitness)", 'sites': [], 'all': True, 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True},
        {'regex': u"([a-zA-Z])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer):
        result = []
        body = re.sub("
.*?
", "", body, flags=re.DOTALL)
        body = re.sub(".*?", "", body, flags=re.DOTALL)
        for rule in FindSpam.rules:
            if rule['all'] != (site in rule['sites']):
                matched_title = re.compile(rule['regex'], re.UNICODE).findall(title)
                matched_username = re.compile(rule['regex'], re.UNICODE).findall(user_name)
                matched_body = re.compile(rule['regex'], re.UNICODE).findall(body)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body']:
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<|endoftext|>


<commit_before># Author: Amund M. Raniseth
# Date: 01.07.2021

import os
from pathlib import Path
import logging
import cellpy
import matplotlib.pyplot as plt
import matplotlib as mpl

#### WORK IN PROGRESS ####

def plot(files, **kwargs):
    plot = Plot(**kwargs) #Initialize plot object
    outpath = handle_outpath(kwargs["outpath"]) #Takes care of the output path

    for file in files:
        # Get the data
        cpobj = cellpy.get(filename = file)     # Initiate cellpy object
        cyc_nums = cpobj.get_cycle_numbers()    # Get ID of all cycles
        color = plot.give_color()               # Get a color for the curves

        if kwargs["galvanostatic_plot"] == True:
            plot_galvanostatic(cpobj, cyc_nums, color, plot, file, outpath)

def plot_galvanostatic(cpobj, cyc_nums, color, plot, file, outpath):

    # Get Pandas DataFrame of pot vs cap from cellpy object
    df = cpobj.get_cap(method="forth-and-forth", label_cycle_number=True, categorical_column=True)

    # Group by cycle and make list of cycle numbers
    cycgrouped = df.groupby("cycle")
    keys = []
    for key, item in cycgrouped:
        keys.append(key)

    # Create the plot obj
    fig, ax = plt.subplots(figsize=(6, 4))

    # Set up colormap and add colorbar
    cmap = mpl.colors.LinearSegmentedColormap.from_list("name", [color, "black"], N=256, gamma=1.0)
    norm = mpl.colors.Normalize(vmin=cyc_nums[0], vmax=cyc_nums[-1])
    fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),label='Cycle')

    # Plot cycles
    for cyc in keys:
        cyccolor = cmap(cyc/keys[-1])
        cyc_df = cycgrouped.get_group(cyc)
        if cyc in cyc_nums:
            ax.plot(cyc_df["capacity"], cyc_df["voltage"], label=str(key), c = cyccolor)

    # Set all plot settings from Plot object
    plot.fix(fig, ax)
    fig.suptitle(file)

    # Save fig
    savepath = outpath + os.path.basename(file).split(".")[0] + "_GC-plot.png"
    print("Saving to: " + savepath)
    fig.savefig(savepath, bbox_inches='tight')

class Plot:
    def __init__(self, **kwargs):
        # Set all kwargs as self.kwarg vars
        for key in kwargs:
            setattr(self, key, kwargs[key])

        self.kwargs = kwargs

        # List of available colors
        self.colors =  ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan' ]

    def give_color(self):
        # Picks the first color from the color list and gives it away
        color = self.colors[0]
        self.colors=self.colors[1:]
        return color

    def fix(self, fig, ax):
        # Applies all the user settings to the fig and axis
        ax.set(
            ylabel = 'Potential [V]',
            xlabel = 'Specific Capacity [mAh/g]',
            #ylim = (2.5,5),
            #xlim = (0, 150),
            #xticks = (np.arange(0, 150), step=20)),
            #yticks = (np.arange(3, 5, step=0.2)),
            )
        ax.tick_params(direction='in', top = 'true', right = 'true')

        # Apply all kwargs to plot

        for kwarg in self.kwargs:
            # Galvanostatic plot details
            if kwarg == "galvanostatic_xlabel":
                ax.set(xlabel = self.kwargs["galvanostatic_xlabel"])
            elif kwarg == "galvanostatic_ylabel":
                ax.set(ylabel = self.kwargs["galvanostatic_ylabel"])
            elif kwarg == "galvanostatic_potlim":
                ax.set(ylim = self.kwargs["galvanostatic_potlim"])
            elif kwarg == "galvanostatic_caplim":
                ax.set(xlim = self.kwargs["galvanostatic_caplim"])

            # General plot details
            elif kwarg == "figsize":
                fig.set_size_inches(self.kwargs["figsize"])
            elif kwarg == "figtitle":
                if type(kwargs["figtitle"]) == str:
                    fig.suptitle(kwargs["figtitle"])

def handle_outpath(dictval):
    if os.path.isdir(dictval):
        return dictval
    elif not os.path.isdir(dictval):
        try:
            os.mkdir(dictval)
            return dictval
        except OSError as e:
            print("Cannot create output directory " + dictval + ". Please make sure you have write permission.")
            print(e)
            exit()<commit_msg>#162 tidy up abit before i go home<commit_after># Author: Amund M. Raniseth
# Date: 01.07.2021

import os
from pathlib import Path
import logging
import cellpy
import matplotlib.pyplot as plt
import matplotlib as mpl

#### WORK IN PROGRESS ####

def plot(files, **kwargs):
    plot = Plot(**kwargs) #Initialize plot object
    outpath = handle_outpath(kwargs["outpath"]) #Takes care of the output path

    for file in files:
        # Get the data
        cpobj = cellpy.get(filename = file)     # Initiate cellpy object
        cyc_nums = cpobj.get_cycle_numbers()    # Get ID of all cycles
        color = plot.give_color()               # Get a color for the curves

        if kwargs["galvanostatic_plot"] == True:
            plot_galvanostatic(cpobj, cyc_nums, color, plot, file, outpath)

def plot_galvanostatic(cpobj, cyc_nums, color, plot, file, outpath):

    # Get Pandas DataFrame of pot vs cap from cellpy object
    df = cpobj.get_cap(method="forth-and-forth", label_cycle_number=True, categorical_column=True)

    # Group by cycle and make list of cycle numbers
    cycgrouped = df.groupby("cycle")
    keys = []
    for key, item in cycgrouped:
        keys.append(key)

    # Create the plot obj
    fig, ax = plt.subplots(figsize=(6, 4))

    # Set up colormap and add colorbar
    cmap = mpl.colors.LinearSegmentedColormap.from_list("name", [color, "black"], N=256, gamma=1.0)
    norm = mpl.colors.Normalize(vmin=cyc_nums[0], vmax=cyc_nums[-1])
    fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),label='Cycle')

    # Plot cycles
    for cyc in keys:
        cyccolor = cmap(cyc/keys[-1])
        cyc_df = cycgrouped.get_group(cyc)
        if cyc in cyc_nums:
            ax.plot(cyc_df["capacity"], cyc_df["voltage"], label=str(key), c = cyccolor)

    # Set all plot settings from Plot object
    plot.fix(fig, ax)
    fig.suptitle(file)

    # Save fig
    savepath = outpath + os.path.basename(file).split(".")[0] + "_GC-plot.png"
    print("Saving to: " + savepath)
    fig.savefig(savepath, bbox_inches='tight')

class Plot:
    def __init__(self, **kwargs):
        # Set all kwargs as self.kwarg vars
        for key in kwargs:
            setattr(self, key, kwargs[key])

        self.kwargs = kwargs

        # List of available colors
        self.colors =  ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan' ]

    def give_color(self):
        # Picks the first color from the color list and gives it away
        color = self.colors[0]
        self.colors=self.colors[1:]
        return color

    def fix(self, fig, ax):
        # Applies kwargs settings and other plot settings

        ## Parameters which could be user defined later
        """
        ax.set(
            xticks = (np.arange(0, 150), step=20)),
            yticks = (np.arange(3, 5, step=0.2)),
            )
        """

        # The params below should always be like this.
        ax.tick_params(direction='in', top = 'true', right = 'true')

        # Apply all kwargs to plot
        for kwarg in self.kwargs:
            # Galvanostatic plot details
            if kwarg == "galvanostatic_xlabel":
                ax.set(xlabel = self.kwargs["galvanostatic_xlabel"])
            elif kwarg == "galvanostatic_ylabel":
                ax.set(ylabel = self.kwargs["galvanostatic_ylabel"])
            elif kwarg == "galvanostatic_potlim":
                ax.set(ylim = self.kwargs["galvanostatic_potlim"])
            elif kwarg == "galvanostatic_caplim":
                ax.set(xlim = self.kwargs["galvanostatic_caplim"])

            # General plot details
            elif kwarg == "figsize":
                fig.set_size_inches(self.kwargs["figsize"])
            elif kwarg == "figtitle":
                if type(kwargs["figtitle"]) == str:
                    fig.suptitle(kwargs["figtitle"])

def handle_outpath(dictval):
    if os.path.isdir(dictval):
        return dictval
    elif not os.path.isdir(dictval):
        try:
            os.mkdir(dictval)
            return dictval
        except OSError as e:
            print("Cannot create output directory " + dictval + ". Please make sure you have write permission.")
            print(e)
            exit()<|endoftext|>