<commit_before># -*- coding: utf-8 -*-

#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

# See LICENSE.txt for licensing terms

import os
from xml.sax.saxutils import escape
from log import log, nodeid
from basenodehandler import NodeHandler
import docutils.nodes
from urlparse import urljoin, urlparse
from reportlab.lib.units import cm
from opt_imports import Paragraph

from image import MyImage, missing

class FontHandler(NodeHandler):
    def get_pre_post(self, client, node, replaceEnt):
        return self.get_font_prefix(client, node, replaceEnt), ''

    def get_font_prefix(self, client, node, replaceEnt):
        return client.styleToFont(self.fontstyle)

class HandleText(NodeHandler, docutils.nodes.Text):
    def gather_elements(self, client, node, style):
        return [Paragraph(client.gather_pdftext(node), style)]

    def get_text(self, client, node, replaceEnt):
        text = node.astext()
        if replaceEnt:
            text = escape(text)
        return text

class HandleStrong(NodeHandler, docutils.nodes.strong):
    pre = ""
    post = ""

class HandleEmphasis(NodeHandler, docutils.nodes.emphasis):
    pre = ""
    post = ""

class HandleLiteral(NodeHandler, docutils.nodes.literal):
    def get_pre_post(self, client, node, replaceEnt):
        pre = '' % client.styles['literal'].fontName
        post = ""
        if not client.styles['literal'].hyphenation:
            pre = '<nobr>' + pre
            post += '</nobr>'
        return pre, post

class HandleSuper(NodeHandler, docutils.nodes.superscript):
    pre = '<super>'
    post = "</super>"

class HandleSub(NodeHandler, docutils.nodes.subscript):
    pre = ''
    post = ""

class HandleTitleReference(FontHandler, docutils.nodes.title_reference):
    fontstyle = 'title_reference'

class HandleReference(NodeHandler, docutils.nodes.reference):
    def get_pre_post(self, client, node, replaceEnt):
        pre, post = '', ''
        uri = node.get('refuri')
        if uri:
            if client.baseurl: # Need to join the uri with the base url
                uri = urljoin(client.baseurl, uri)

            if urlparse(uri)[0] and client.inlinelinks:
                # external inline reference
                post = u' (%s)' % uri
            else:
                # A plain old link
                pre += u'' %\
                    (uri, client.styles.linkColor)
                post = '' + post
        else:
            uri = node.get('refid')
            if uri:
                pre += u'' %\
                    (uri, client.styles.linkColor)
                post = '' + post
        return pre, post

class HandleOptions(HandleText, docutils.nodes.option_string, docutils.nodes.option_argument):
    pass

class HandleSysMessage(HandleText, docutils.nodes.system_message, docutils.nodes.problematic):
    pre = ''
    post = ""

    def gather_elements(self, client, node, style):
        # FIXME show the error in the document, red, whatever
        # log.warning("Problematic node %s", node.astext())
        return []

class HandleGenerated(HandleText, docutils.nodes.generated):
    pass

class HandleImage(NodeHandler, docutils.nodes.image):
    def gather_elements(self, client, node, style):
        # FIXME: handle class,target,alt, check align
        imgname = os.path.join(client.basedir,str(node.get("uri")))
        w, h, kind = MyImage.size_for_node(node, client=client)
        node.elements = [MyImage(filename=imgname, height=h, width=w,
                    kind=kind, client=client)]
        alignment = node.get('align', 'CENTER').upper()
        if alignment in ('LEFT', 'CENTER', 'RIGHT'):
            node.elements[0].image.hAlign = alignment
        # Image flowables don't support valign (makes no sense for them?)
        # elif alignment in ('TOP','MIDDLE','BOTTOM'):
        #    i.vAlign = alignment
        return node.elements

    def get_text(self, client, node, replaceEnt):
        # First see if the image file exists, or else,
        # use image-missing.png
        uri=node.get('uri')
        w, h, kind = MyImage.size_for_node(node, client=client)
        alignment=node.get('align', 'CENTER').lower()
        if alignment in ('top', 'middle', 'bottom'):
            align='valign="%s"'%alignment
        else:
            align=''
        uri=os.path.join(client.basedir,uri)
        # TODO: inline images don't support SVG, vectors and PDF,
        #       which may be surprising. So, work on converting them
        #       previous to passing to reportlab.
	uri=MyImage.raster(uri, client)
        return ''%\
            (uri, w, h, align)

class HandleFootRef(NodeHandler, docutils.nodes.footnote_reference):
    def get_text(self, client, node, replaceEnt):
        # TODO: when used in Sphinx, all footnotes are autonumbered
        anchors=''
        for i in node['ids']:
            if i not in client.targets:
                anchors+='' % i
                client.targets.append(i)
        return u'%s<super>%s</super>'%\
            (anchors, '#' + node.astext(),
                client.styles.linkColor, node.astext())

class HandleCiteRef(NodeHandler, docutils.nodes.citation_reference):
    def get_text(self, client, node, replaceEnt):
        anchors=''
        for i in node['ids']:
            if i not in client.targets:
                anchors +='' % i
                client.targets.append(i)
        return u'%s[%s]'%\
            (anchors, '#' + node.astext(),
                client.styles.linkColor, node.astext())

class HandleTarget(NodeHandler, docutils.nodes.target):
    def gather_elements(self, client, node, style):
        if 'refid' in node:
            client.pending_targets.append(node['refid'])
        return client.gather_elements(node, style)

    def get_text(self, client, node, replaceEnt):
        text = client.gather_pdftext(node)
        if replaceEnt:
            text = escape(text)
        return text

    def get_pre_post(self, client, node, replaceEnt):
        pre = ''
        if node['ids'][0] not in client.targets:
            pre = u'' % node['ids'][0]
            client.targets.append(node['ids'][0])
        return pre, ''

class HandleInline(NodeHandler, docutils.nodes.inline):
    def get_pre_post(self, client, node, replaceEnt):
        ftag = client.styleToFont(node['classes'][0])
        if ftag:
            return ftag, ''
        return '', ''
<commit_msg>Stray tab<commit_after># -*- coding: utf-8 -*-

#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

# See LICENSE.txt for licensing terms

import os
from xml.sax.saxutils import escape
from log import log, nodeid
from basenodehandler import NodeHandler
import docutils.nodes
from urlparse import urljoin, urlparse
from reportlab.lib.units import cm
from opt_imports import Paragraph

from image import MyImage, missing

class FontHandler(NodeHandler):
    def get_pre_post(self, client, node, replaceEnt):
        return self.get_font_prefix(client, node, replaceEnt), ''

    def get_font_prefix(self, client, node, replaceEnt):
        return client.styleToFont(self.fontstyle)

class HandleText(NodeHandler, docutils.nodes.Text):
    def gather_elements(self, client, node, style):
        return [Paragraph(client.gather_pdftext(node), style)]

    def get_text(self, client, node, replaceEnt):
        text = node.astext()
        if replaceEnt:
            text = escape(text)
        return text

class HandleStrong(NodeHandler, docutils.nodes.strong):
    pre = ""
    post = ""

class HandleEmphasis(NodeHandler, docutils.nodes.emphasis):
    pre = ""
    post = ""

class HandleLiteral(NodeHandler, docutils.nodes.literal):
    def get_pre_post(self, client, node, replaceEnt):
        pre = '' % client.styles['literal'].fontName
        post = ""
        if not client.styles['literal'].hyphenation:
            pre = '<nobr>' + pre
            post += '</nobr>'
        return pre, post

class HandleSuper(NodeHandler, docutils.nodes.superscript):
    pre = '<super>'
    post = "</super>"

class HandleSub(NodeHandler, docutils.nodes.subscript):
    pre = ''
    post = ""

class HandleTitleReference(FontHandler, docutils.nodes.title_reference):
    fontstyle = 'title_reference'

class HandleReference(NodeHandler, docutils.nodes.reference):
    def get_pre_post(self, client, node, replaceEnt):
        pre, post = '', ''
        uri = node.get('refuri')
        if uri:
            if client.baseurl: # Need to join the uri with the base url
                uri = urljoin(client.baseurl, uri)

            if urlparse(uri)[0] and client.inlinelinks:
                # external inline reference
                post = u' (%s)' % uri
            else:
                # A plain old link
                pre += u'' %\
                    (uri, client.styles.linkColor)
                post = '' + post
        else:
            uri = node.get('refid')
            if uri:
                pre += u'' %\
                    (uri, client.styles.linkColor)
                post = '' + post
        return pre, post

class HandleOptions(HandleText, docutils.nodes.option_string, docutils.nodes.option_argument):
    pass

class HandleSysMessage(HandleText, docutils.nodes.system_message, docutils.nodes.problematic):
    pre = ''
    post = ""

    def gather_elements(self, client, node, style):
        # FIXME show the error in the document, red, whatever
        # log.warning("Problematic node %s", node.astext())
        return []

class HandleGenerated(HandleText, docutils.nodes.generated):
    pass

class HandleImage(NodeHandler, docutils.nodes.image):
    def gather_elements(self, client, node, style):
        # FIXME: handle class,target,alt, check align
        imgname = os.path.join(client.basedir,str(node.get("uri")))
        w, h, kind = MyImage.size_for_node(node, client=client)
        node.elements = [MyImage(filename=imgname, height=h, width=w,
                    kind=kind, client=client)]
        alignment = node.get('align', 'CENTER').upper()
        if alignment in ('LEFT', 'CENTER', 'RIGHT'):
            node.elements[0].image.hAlign = alignment
        # Image flowables don't support valign (makes no sense for them?)
        # elif alignment in ('TOP','MIDDLE','BOTTOM'):
        #    i.vAlign = alignment
        return node.elements

    def get_text(self, client, node, replaceEnt):
        # First see if the image file exists, or else,
        # use image-missing.png
        uri=node.get('uri')
        w, h, kind = MyImage.size_for_node(node, client=client)
        alignment=node.get('align', 'CENTER').lower()
        if alignment in ('top', 'middle', 'bottom'):
            align='valign="%s"'%alignment
        else:
            align=''
        uri=os.path.join(client.basedir,uri)
        # TODO: inline images don't support SVG, vectors and PDF,
        #       which may be surprising. So, work on converting them
        #       previous to passing to reportlab.
        uri=MyImage.raster(uri, client)
        return ''%\
            (uri, w, h, align)

class HandleFootRef(NodeHandler, docutils.nodes.footnote_reference):
    def get_text(self, client, node, replaceEnt):
        # TODO: when used in Sphinx, all footnotes are autonumbered
        anchors=''
        for i in node['ids']:
            if i not in client.targets:
                anchors+='' % i
                client.targets.append(i)
        return u'%s<super>%s</super>'%\
            (anchors, '#' + node.astext(),
                client.styles.linkColor, node.astext())

class HandleCiteRef(NodeHandler, docutils.nodes.citation_reference):
    def get_text(self, client, node, replaceEnt):
        anchors=''
        for i in node['ids']:
            if i not in client.targets:
                anchors +='' % i
                client.targets.append(i)
        return u'%s[%s]'%\
            (anchors, '#' + node.astext(),
                client.styles.linkColor, node.astext())

class HandleTarget(NodeHandler, docutils.nodes.target):
    def gather_elements(self, client, node, style):
        if 'refid' in node:
            client.pending_targets.append(node['refid'])
        return client.gather_elements(node, style)

    def get_text(self, client, node, replaceEnt):
        text = client.gather_pdftext(node)
        if replaceEnt:
            text = escape(text)
        return text

    def get_pre_post(self, client, node, replaceEnt):
        pre = ''
        if node['ids'][0] not in client.targets:
            pre = u'' % node['ids'][0]
            client.targets.append(node['ids'][0])
        return pre, ''

class HandleInline(NodeHandler, docutils.nodes.inline):
    def get_pre_post(self, client, node, replaceEnt):
        ftag = client.styleToFont(node['classes'][0])
        if ftag:
            return ftag, ''
        return '', ''
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php
#
# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
# Copyright 2014 Hartmut Goebel <h.goebel@crazy-compilers.com>
#
"""
little helper to get the proper ElementTree package
"""

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
    except ImportError:
        # this seems to be necessary with the python2.5 on the Maemo platform
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                raise ImportError("ElementTree: no ElementTree module found, "
                                  "critical error")

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def new_encode_entity(text, pattern=utf8_escape):

    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        """
        map reserved and non-ascii characters to numerical entities
        """
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)

    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(
                escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(
                utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.
if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

def namespace_map_update(namespaces):
    for uri, prefix in namespaces.items():
        elementtree.ElementTree.register_namespace(prefix, uri)

class ElementInterface(elementtree.ElementTree._ElementInterface): pass

def indent(elem, level=0):
    """
    generate pretty looking XML, based upon:
    http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level + 1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8", dump_invalid_data=False):
    try:
        parser = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        parser = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    # :fixme: This must be handled where twisted.web is fetching the data
    if isinstance(data, (list, tuple)):
        data = data[0]

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00', '')
    try:
        parser.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        parser.close()
        raise
    else:
        return ET.ElementTree(parser.close())

def qname(tag, ns=None):
    if not ns:
        return tag
    return "{%s}%s" % (ns, tag)
<commit_msg>Add two helper functions for creating a subelement with text content.<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php
#
# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
# Copyright 2014 Hartmut Goebel <h.goebel@crazy-compilers.com>
#
"""
little helper to get the proper ElementTree package
"""

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
    except ImportError:
        # this seems to be necessary with the python2.5 on the Maemo platform
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                raise ImportError("ElementTree: no ElementTree module found, "
                                  "critical error")

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def new_encode_entity(text, pattern=utf8_escape):

    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        """
        map reserved and non-ascii characters to numerical entities
        """
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)

    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(
                escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(
                utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.
if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

def namespace_map_update(namespaces):
    for uri, prefix in namespaces.items():
        elementtree.ElementTree.register_namespace(prefix, uri)

class ElementInterface(elementtree.ElementTree._ElementInterface): pass

def indent(elem, level=0):
    """
    generate pretty looking XML, based upon:
    http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level + 1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8", dump_invalid_data=False):
    try:
        parser = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        parser = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    # :fixme: This must be handled where twisted.web is fetching the data
    if isinstance(data, (list, tuple)):
        data = data[0]

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00', '')
    try:
        parser.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        parser.close()
        raise
    else:
        return ET.ElementTree(parser.close())

def qname(tag, ns=None):
    if not ns:
        return tag
    return "{%s}%s" % (ns, tag)

def textElement(parent, tag, namespace, text):
    """Create a subelement with text content."""
    elem = ET.SubElement(parent, qname(tag, namespace))
    elem.text = text
    return elem

def textElementIfNotNone(parent, tag, namespace, text):
    """If text is not none, create a subelement with text content."""
    if text is None:
        return
    return textElement(parent, tag, namespace, str(text))
<|endoftext|>


<commit_before>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase

class Comicdata(ComicDataBase):
    name = 'Zelda'
    language = 'no'
    url = 'http://www.dagbladet.no/tegneserie/zelda/'
    start_date = '2012-06-07'
    rights = 'Lina Neidestam'

class Crawler(CrawlerBase):
    history_capable_days = 30
    time_zone = 1

    def crawl(self, pub_date):
        page_url = 'http://www.dagbladet.no/tegneserie/zelda/?%s' % (
            self.date_to_epoch(pub_date),)
        page = self.parse_page(page_url)
        url = page.src('img#zelda-stripe')
        return CrawlerImage(url)
<commit_msg>Fix typo<commit_after>from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase

class ComicData(ComicDataBase):
    name = 'Zelda'
    language = 'no'
    url = 'http://www.dagbladet.no/tegneserie/zelda/'
    start_date = '2012-06-07'
    rights = 'Lina Neidestam'

class Crawler(CrawlerBase):
    history_capable_days = 30
    time_zone = 1

    def crawl(self, pub_date):
        page_url = 'http://www.dagbladet.no/tegneserie/zelda/?%s' % (
            self.date_to_epoch(pub_date),)
        page = self.parse_page(page_url)
        url = page.src('img#zelda-stripe')
        return CrawlerImage(url)
<|endoftext|>


<commit_before>import logging

from comics.comics import get_comic_module_names, get_comic_module
from comics.meta.exceptions import MetaError

logger = logging.getLogger('comics.meta.command')

class MetaLoader(object):
    def __init__(self, options):
        self.comic_slugs = self._get_comic_slugs(options)

    def start(self):
        for comic_slug in self.comic_slugs:
            logger.info('Loading meta data for %s', comic_slug)
            self._try_load_comic_meta(comic_slug)

    def stop(self):
        pass

    def _get_comic_slugs(self, options):
        comic_slugs = options.get('comic_slugs', None)
        if comic_slugs is None or len(comic_slugs) == 0:
            logger.debug('Load targets: all comics')
            return get_comic_module_names()
        else:
            logger.debug('Load targets: %s', comic_slugs)
            return comic_slugs

    def _try_load_comic_meta(self, comic_slug):
        try:
            meta = self._get_meta(comic_slug)
            self._load_meta(meta)
        except MetaError, error:
            logger.error(error)
        except Exception, error:
            logger.exception(error)

    def _get_meta(self, comic_slug):
        logger.debug('Importing comic module for %s', comic_slug)
        comic_module = get_comic_module(comic_slug)
        if not hasattr(comic_module, 'Meta'):
            raise MetaError('%s does not have a Meta class' %
                comic_module.__name__)
        return comic_module.Meta()

    def _load_meta(self, meta):
        logger.debug('Syncing comic meta data with database')
        meta.create_comic()
<commit_msg>Do not load meta for inactive comics unless explicitly requested<commit_after>import logging

from comics.comics import get_comic_module_names, get_comic_module
from comics.meta.exceptions import MetaError

logger = logging.getLogger('comics.meta.command')

class MetaLoader(object):
    def __init__(self, options):
        self.include_inactive = self._get_include_inactive(options)
        self.comic_slugs = self._get_comic_slugs(options)

    def start(self):
        for comic_slug in self.comic_slugs:
            logger.info('Loading meta data for %s', comic_slug)
            self._try_load_comic_meta(comic_slug)

    def stop(self):
        pass

    def _get_include_inactive(self, options):
        comic_slugs = options.get('comic_slugs', None)
        if comic_slugs is None or len(comic_slugs) == 0:
            logger.debug('Excluding inactive comics')
            return False
        else:
            logger.debug('Including inactive comics')
            return True

    def _get_comic_slugs(self, options):
        comic_slugs = options.get('comic_slugs', None)
        if comic_slugs is None or len(comic_slugs) == 0:
            logger.debug('Load targets: all comics')
            return get_comic_module_names()
        else:
            logger.debug('Load targets: %s', comic_slugs)
            return comic_slugs

    def _try_load_comic_meta(self, comic_slug):
        try:
            meta = self._get_meta(comic_slug)
            if self._should_load_meta(meta):
                self._load_meta(meta)
            else:
                logger.debug('Skipping inactive comic')
        except MetaError, error:
            logger.error(error)
        except Exception, error:
            logger.exception(error)

    def _get_meta(self, comic_slug):
        logger.debug('Importing comic module for %s', comic_slug)
        comic_module = get_comic_module(comic_slug)
        if not hasattr(comic_module, 'Meta'):
            raise MetaError('%s does not have a Meta class' %
                comic_module.__name__)
        return comic_module.Meta()

    def _should_load_meta(self, meta):
        return self.include_inactive or meta.active

    def _load_meta(self, meta):
        logger.debug('Syncing comic meta data with database')
        meta.create_comic()
<|endoftext|>


<commit_before>import ha_engine.ha_infra as common

LOG = common.ha_logging(__name__)

class BaseRunner(object):
    def __init__(self, input_args):
        self.ha_report = []
        self.input_args = {}

        if input_args:
            self.set_input_arguments(input_args)

    def set_input_arguments(self, input_args):

        self.input_args = input_args
        LOG.info("Self, input %s " , str(self.input_args))

    def get_input_arguments(self):
        return self.input_args

    def get_input_arguments(self):
        return self.input_args

    def execute(self, sync=None, finish_execution=None):
        raise NotImplementedError('Subclass should implement this method')

    def setup(self):
        raise NotImplementedError('Subclass should implement this method')

    def teardown(self):
        raise NotImplementedError('Subclass should implement this method')

    def set_report(self):
        raise NotImplementedError('Subclass should implement this method')

    def display_report(self):
        raise NotImplementedError('Subclass should implement this method')

    def stable(self):
        raise NotImplementedError('Subclass should implement this method')

    def is_module_exeution_completed(self, finish_exection):
        raise NotImplementedError('Subclass should implement this method')

    def get_ha_interval(self):
        return self.ha_interval<commit_msg>Clean base runner methods<commit_after>import ha_engine.ha_infra as common

LOG = common.ha_logging(__name__)

class BaseRunner(object):
    def __init__(self, input_args):
        self.ha_report = []
        self.input_args = {}

        if input_args:
            self.set_input_arguments(input_args)

    def set_input_arguments(self, input_args):

        self.input_args = input_args
        LOG.info("Self, input %s " , str(self.input_args))

    def get_input_arguments(self):
        return self.input_args

    def get_input_arguments(self):
        return self.input_args

    def execute(self, sync=None, finish_execution=None):
        raise NotImplementedError('Subclass should implement this method')

    def setup(self):
        raise NotImplementedError('Subclass should implement this method')

    def teardown(self):
        raise NotImplementedError('Subclass should implement this method')

    def is_module_exeution_completed(self, finish_exection):
        raise NotImplementedError('Subclass should implement this method')

    def get_ha_interval(self):
        return self.ha_interval<|endoftext|>


<commit_before>import pylast
from lib.command import Command
from lib.utils import escape_telegram_html

ADD_STRINGS = [
    "-a", "-add", "--add",
    "-s", "-set", "--set"
]

class LastFMCommand(Command):
    name = 'lastfm'
    aliases = ['np', 'nowplaying']
    description = 'Post your currently playing song.'
    has_database = True

    def run(self, message, args):
        if not self.config:
            self.reply(message, 'last.fm command is not configured!')
            self.logger.error('last.fm API key is not configured.')
            return

        if not message.from_user.username:
            self.reply(message, 'You do not have a Telegram username I can use with last.fm!')
            return

        network = pylast.LastFMNetwork(api_key=self.config['api_key'])
        user = message.from_user
        if args and args[0] in ADD_STRINGS:
            if len(args) == 1:
                self.reply(message, 'Please provide a username. (/np -s <username>)')
                return
            username = args[1].strip()
            user_exists = False
            try:
                network.get_user(username).get_id()
                user_exists = True
            except pylast.WSError:
                pass
            if user_exists:
                self.database.set_user_value(user, 'username', username)
                self.reply(message, 'last.fm username set.')
            else:
                self.reply(message, 'No such last.fm user. Are you trying to trick me? :^)')
            return

        username = self.database.get_user_value(user, 'username')
        if not username:
            self.reply(message, 'You have no last.fm username set. Please set one with /np -s <username>')
            return

        user = network.get_user(username)
        current_track = user.get_now_playing()
        if not current_track:
            reply = '{0} is not listening to anything right now.'.format(
                escape_telegram_html(username))
            self.reply(message, reply, parse_mode='HTML')
            return

        trackinfo = '{1} - {2}'.format(
            escape_telegram_html(current_track.get_url()),
            escape_telegram_html(current_track.get_artist().get_name()),
            escape_telegram_html(current_track.get_title()))

        self.reply(message, '{0} is now listening to {1}.'.format(
            escape_telegram_html(username), trackinfo), parse_mode='HTML')
<commit_msg>Update ADD_STRINGS<commit_after>import pylast
from lib.command import Command
from lib.utils import escape_telegram_html

SET_STRINGS = [
    '-s', '-set', '--set', 'set'
]

class LastFMCommand(Command):
    name = 'lastfm'
    aliases = ['np', 'nowplaying']
    description = 'Post your currently playing song.'
    has_database = True

    def run(self, message, args):
        if not self.config:
            self.reply(message, 'last.fm command is not configured!')
            self.logger.error('last.fm API key is not configured.')
            return

        if not message.from_user.username:
            self.reply(message, 'You do not have a Telegram username I can use with last.fm!')
            return

        network = pylast.LastFMNetwork(api_key=self.config['api_key'])
        user = message.from_user
        if args and args[0] in SET_STRINGS:
            if len(args) == 1:
                self.reply(message, 'Please provide a username. (/np -s <username>)')
                return
            username = args[1].strip()
            user_exists = False
            try:
                network.get_user(username).get_id()
                user_exists = True
            except pylast.WSError:
                pass
            if user_exists:
                self.database.set_user_value(user, 'username', username)
                self.reply(message, 'last.fm username set.')
            else:
                self.reply(message, 'No such last.fm user. Are you trying to trick me? :^)')
            return

        username = self.database.get_user_value(user, 'username')
        if not username:
            self.reply(message, 'You have no last.fm username set. Please set one with /np -s <username>')
            return

        user = network.get_user(username)
        current_track = user.get_now_playing()
        if not current_track:
            reply = '{0} is not listening to anything right now.'.format(
                escape_telegram_html(username))
            self.reply(message, reply, parse_mode='HTML')
            return

        trackinfo = '{1} - {2}'.format(
            escape_telegram_html(current_track.get_url()),
            escape_telegram_html(current_track.get_artist().get_name()),
            escape_telegram_html(current_track.get_title()))

        self.reply(message, '{0} is now listening to {1}.'.format(
            escape_telegram_html(username), trackinfo), parse_mode='HTML')
<|endoftext|>


<commit_before>import os
import re
import sublime
import sublime_plugin
import subprocess

TEXT_ENCODING = 'utf-8'

class CommandBase():
    def get_backend(self):
        """Get the backend (Git or Mercurial) for the current project.
        """
        working_directory = self.get_directory()
        for backend in [GitBackend, MercurialBackend]:
            try:
                subprocess.check_output('%s status' % backend.command,
                                        cwd=working_directory, shell=True)
                return backend(working_directory)
            except subprocess.CalledProcessError:
                pass

        raise SublimeBucketError('Unable to find a Git/Hg repository')

    def get_directory(self):
        """Get the open directory containing the current file.
        """
        full_path = self.view.file_name()
        for folder in self.view.window().folders():
            if full_path.startswith(folder):
                return folder

    def get_file_path(self):
        """Get the path to the current file, relative to the repository root.
        """
        file_path = self.view.file_name()[len(self.get_directory()):]
        if file_path.startswith('/'):
            file_path = file_path[1:]
        return file_path

    def get_line_ranges(self):
        """Get the list of currently selected line ranges, in start:end format.
        """
        ranges = []

        for (start, end) in self.view.sel():
            # Sublime provides 0-based line numbers but Bitbucket line numbers
            # start with 1.
            first_line = self.view.rowcol(start)[0] + 1
            last_line = self.view.rowcol(end)[0] + 1

            if first_line == last_line:
                ranges.append(str(first_line))
            else:
                # Fix ordering in case selection was made bottom-up.
                ranges.append('%d:%d' % (min(first_line, last_line),
                                         max(first_line, last_line)))

        return ranges

    def get_current_line(self):
        """Get the 1-based line number of the (first) currently selected line.
        """
        start, end = self.view.sel()[0]
        row, col = self.view.rowcol(start)
        return row + 1

class OpenInBitbucketCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            remote_match = backend.find_bitbucket_remote_match()
            url = '%(host)s/%(repo)s/src/%(branch)s/%(path)s#%(hash)s' % {
                'host': 'https://' + remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'branch': backend.find_current_revision(),
                'path': self.get_file_path(),
                'hash': '%s-%s' % (os.path.basename(self.view.file_name()),
                                   ','.join(self.get_line_ranges()))
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception as e:
            print(e)
            sublime.error_message('Encountered an unexpected error')

class OpenBitbucketChangesetCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            remote_match = backend.find_bitbucket_remote_match()
            url = '%(host)s/%(repo)s/commits/%(hash)s#chg-%(file)s' % {
                'host': 'https://' + remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'hash': backend.find_selected_revision(
                    self.get_file_path(), self.get_current_line()),
                'file': self.get_file_path()
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception as e:
            print(e)
            sublime.error_message('Encountered an unexpected error')

class FindBitbucketPullRequestCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            target_revision = backend.find_selected_revision(
                self.get_file_path(), self.get_current_line())
            pull_request_id = backend.get_pull_request_id(target_revision)
            remote_match = backend.find_bitbucket_remote_match()
            url = ('https://%(host)s/%(repo)s/pull-requests/%(id)d/diff'
                   '#chg-%(file)s') % {
                'host': remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'id': pull_request_id,
                'file': self.get_file_path()
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception as e:
            print(e)
            sublime.error_message('Encountered an unexpected error')

class BackendBase():
    def __init__(self, cwd):
        self.cwd = cwd

    @property
    def settings(self):
        return sublime.load_settings('Bitbucket.sublime-settings')

    @property
    def bitbucket_hosts(self):
        custom_hosts = self.settings.get('bitbucket_hosts', [])
        return list(set(['bitbucket.org'] + custom_hosts))

    def find_bitbucket_remote_match(self):
        """Get a regex match of the first remote containing a Bitbucket host.

        Returns an _sre.SRE_MATCH object w/ `string` attribute referring to the
        full remote string.
        """
        remotes = self.get_remote_list()

        for host in self.bitbucket_hosts:
            bitbucket_pattern = (r'(?P<host>%s)[:/](?P<repo>[\w\-]+/[\w\-]+)'
                                 r'(?:\.git)?') % host
            for remote in remotes:
                remote_match = re.search(bitbucket_pattern, remote)
                if remote_match:
                    return remote_match

        raise SublimeBucketError('Unable to find a remote matching: %r' %
                                 self.bitbucket_hosts)

    def find_bitbucket_remote(self):
        """Get the name of the remote (e.g. 'origin') pointing to Bitbucket.
        """
        remote_match = self.find_bitbucket_remote_match()

        # For both Git and Hg the remote name is the first token in the string.
        return re.split(r'\s+', remote_match.string, maxsplit=1)[0]

    def find_current_revision(self):
        """Get the hash of the commit/changeset that's currently checked out.
        """
        raise NotImplementedError

    def get_default_branch(self):
        """Get the default branch for the current repo.
        """
        raise NotImplementedError

    def find_selected_revision(self, file_path, current_line):
        """Get the hash of the revision associated with the current line.
        """
        raise NotImplementedError

    def get_pull_request_id(self, target_revision):
        """Get the Bitbucket pull request ID where a given change was merged.

        This method invokes `git rev-list` to compute the merge commit for Git
        repos. For Mercurial repos it uses `hg log -r` to query for the
        changeset using Mercurial's revset syntax.

        This method then uses `git show` for Git repos and `hg log -r` for
        Mercurial repos and scans the commit message for an explicit mention of
        a pull request, which is populated by default in the Bitbucket UI.

        This won't work if the author of the PR wrote a custom commit message
        without mentioning the PR.
        """
        raise NotImplementedError

    def _exec(self, command):
        """Execute command with cwd set to the project path and shell=True.
        """
        output = subprocess.check_output(command, cwd=self.cwd, shell=True)
        return output.decode(TEXT_ENCODING)

class GitBackend(BackendBase):
    command = 'git'

    def get_remote_list(self):
        return self._exec('git remote -v').splitlines()

    def find_current_revision(self):
        info = self._exec('git show HEAD').splitlines()
        for line in info:
            revision_match = re.search(r'commit (\w+)', line)
            if revision_match:
                return revision_match.group(1)

        raise SublimeBucketError('Unable to get the current revision')

    def find_selected_revision(self, file_path, current_line):
        output = self._exec(
            'git blame -L %d,%d %s' % (current_line, current_line, file_path))
        return re.match(r'^(\w+)', output).group(1)

    def get_default_branch(self):
        remote = self.find_bitbucket_remote()
        return self._exec('git rev-parse --abbrev-ref refs/remotes/%s/HEAD'
                          % remote).strip()

    def get_pull_request_id(self, target_revision):
        default_branch = self.get_default_branch()
        revspec = '%s..%s' % (target_revision, default_branch)

        # First find the merge commit where the given commit was merged into
        # the default branch.
        ancestry_path = self._exec('git rev-list %s --ancestry-path' %
                                   revspec).splitlines()
        first_parent = self._exec('git rev-list %s --first-parent' %
                                  revspec).splitlines()
        common = set(ancestry_path) & set(first_parent)
        merge_revision = next(rev for rev in reversed(ancestry_path)
                              if rev in common)

        # Look at the commit message for the merge for a reference to the PR.
        info = self._exec('git show --oneline %s' % merge_revision)
        pull_request_match = re.search(r'pull request #(\d+)', info)
        if pull_request_match:
            return int(pull_request_match.group(1))

        raise SublimeBucketError('Unable to determine the pull request where '
                                 'the commit was merged')

class MercurialBackend(BackendBase):
    command = 'hg'

    def get_remote_list(self):
        return self._exec('hg paths').splitlines()

    def find_current_revision(self):
        return self._exec('hg id -i').strip()

    def find_selected_revision(self, file_path, current_line):
        output = self._exec(
            'hg annotate -c %s | head -n %d | tail -n 1' % (file_path,
                                                            current_line))
        return re.match(r'^(\w+)', output).group(1)

    def get_default_branch(self):
        return 'default'

    def get_pull_request_id(self, target_revision):
        log_output = self._exec(
            'hg log -r "first(descendants(%s) and desc(\'pull request\') '
            'and merge())"' % target_revision)
        pull_request_match = re.search(r'pull request #(\d+)', log_output)
        if pull_request_match:
            return int(pull_request_match.group(1))

        raise SublimeBucketError('Unable to determine the pull request where '
                                 'the changeset was merged')

class SublimeBucketError(Exception):
    pass
<commit_msg>updated exception handling to print entire stack trace<commit_after>import os
import re
import sublime
import sublime_plugin
import subprocess
import traceback

TEXT_ENCODING = 'utf-8'

class CommandBase():
    def get_backend(self):
        """Get the backend (Git or Mercurial) for the current project.
        """
        working_directory = self.get_directory()
        for backend in [GitBackend, MercurialBackend]:
            try:
                subprocess.check_output('%s status' % backend.command,
                                        cwd=working_directory, shell=True)
                return backend(working_directory)
            except subprocess.CalledProcessError:
                pass

        raise SublimeBucketError('Unable to find a Git/Hg repository')

    def get_directory(self):
        """Get the open directory containing the current file.
        """
        full_path = self.view.file_name()
        for folder in self.view.window().folders():
            if full_path.startswith(folder):
                return folder

    def get_file_path(self):
        """Get the path to the current file, relative to the repository root.
        """
        file_path = self.view.file_name()[len(self.get_directory()):]
        if file_path.startswith('/'):
            file_path = file_path[1:]
        return file_path

    def get_line_ranges(self):
        """Get the list of currently selected line ranges, in start:end format.
        """
        ranges = []

        for (start, end) in self.view.sel():
            # Sublime provides 0-based line numbers but Bitbucket line numbers
            # start with 1.
            first_line = self.view.rowcol(start)[0] + 1
            last_line = self.view.rowcol(end)[0] + 1

            if first_line == last_line:
                ranges.append(str(first_line))
            else:
                # Fix ordering in case selection was made bottom-up.
                ranges.append('%d:%d' % (min(first_line, last_line),
                                         max(first_line, last_line)))

        return ranges

    def get_current_line(self):
        """Get the 1-based line number of the (first) currently selected line.
        """
        start, end = self.view.sel()[0]
        row, col = self.view.rowcol(start)
        return row + 1

class OpenInBitbucketCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            remote_match = backend.find_bitbucket_remote_match()
            url = '%(host)s/%(repo)s/src/%(branch)s/%(path)s#%(hash)s' % {
                'host': 'https://' + remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'branch': backend.find_current_revision(),
                'path': self.get_file_path(),
                'hash': '%s-%s' % (os.path.basename(self.view.file_name()),
                                   ','.join(self.get_line_ranges()))
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception:
            traceback.print_exc()
            sublime.error_message('Encountered an unexpected error')

class OpenBitbucketChangesetCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            remote_match = backend.find_bitbucket_remote_match()
            url = '%(host)s/%(repo)s/commits/%(hash)s#chg-%(file)s' % {
                'host': 'https://' + remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'hash': backend.find_selected_revision(
                    self.get_file_path(), self.get_current_line()),
                'file': self.get_file_path()
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception:
            traceback.print_exc()
            sublime.error_message('Encountered an unexpected error')

class FindBitbucketPullRequestCommand(CommandBase, sublime_plugin.TextCommand):
    def run(self, edit):
        backend = self.get_backend()

        try:
            target_revision = backend.find_selected_revision(
                self.get_file_path(), self.get_current_line())
            pull_request_id = backend.get_pull_request_id(target_revision)
            remote_match = backend.find_bitbucket_remote_match()
            url = ('https://%(host)s/%(repo)s/pull-requests/%(id)d/diff'
                   '#chg-%(file)s') % {
                'host': remote_match.group('host'),
                'repo': remote_match.group('repo'),
                'id': pull_request_id,
                'file': self.get_file_path()
            }
            subprocess.call(['open', url])
        except SublimeBucketError as e:
            sublime.error_message(str(e))
        except Exception:
            traceback.print_exc()
            sublime.error_message('Encountered an unexpected error')

class BackendBase():
    def __init__(self, cwd):
        self.cwd = cwd

    @property
    def settings(self):
        return sublime.load_settings('Bitbucket.sublime-settings')

    @property
    def bitbucket_hosts(self):
        custom_hosts = self.settings.get('bitbucket_hosts', [])
        return list(set(['bitbucket.org'] + custom_hosts))

    def find_bitbucket_remote_match(self):
        """Get a regex match of the first remote containing a Bitbucket host.

        Returns an _sre.SRE_MATCH object w/ `string` attribute referring to the
        full remote string.
        """
        remotes = self.get_remote_list()

        for host in self.bitbucket_hosts:
            bitbucket_pattern = (r'(?P<host>%s)[:/](?P<repo>[\w\-]+/[\w\-]+)'
                                 r'(?:\.git)?') % host
            for remote in remotes:
                remote_match = re.search(bitbucket_pattern, remote)
                if remote_match:
                    return remote_match

        raise SublimeBucketError('Unable to find a remote matching: %r' %
                                 self.bitbucket_hosts)

    def find_bitbucket_remote(self):
        """Get the name of the remote (e.g. 'origin') pointing to Bitbucket.
        """
        remote_match = self.find_bitbucket_remote_match()

        # For both Git and Hg the remote name is the first token in the string.
        return re.split(r'\s+', remote_match.string, maxsplit=1)[0]

    def find_current_revision(self):
        """Get the hash of the commit/changeset that's currently checked out.
        """
        raise NotImplementedError

    def get_default_branch(self):
        """Get the default branch for the current repo.
        """
        raise NotImplementedError

    def find_selected_revision(self, file_path, current_line):
        """Get the hash of the revision associated with the current line.
        """
        raise NotImplementedError

    def get_pull_request_id(self, target_revision):
        """Get the Bitbucket pull request ID where a given change was merged.

        This method invokes `git rev-list` to compute the merge commit for Git
        repos. For Mercurial repos it uses `hg log -r` to query for the
        changeset using Mercurial's revset syntax.

        This method then uses `git show` for Git repos and `hg log -r` for
        Mercurial repos and scans the commit message for an explicit mention of
        a pull request, which is populated by default in the Bitbucket UI.

        This won't work if the author of the PR wrote a custom commit message
        without mentioning the PR.
        """
        raise NotImplementedError

    def _exec(self, command):
        """Execute command with cwd set to the project path and shell=True.
        """
        output = subprocess.check_output(command, cwd=self.cwd, shell=True)
        return output.decode(TEXT_ENCODING)

class GitBackend(BackendBase):
    command = 'git'

    def get_remote_list(self):
        return self._exec('git remote -v').splitlines()

    def find_current_revision(self):
        info = self._exec('git show HEAD').splitlines()
        for line in info:
            revision_match = re.search(r'commit (\w+)', line)
            if revision_match:
                return revision_match.group(1)

        raise SublimeBucketError('Unable to get the current revision')

    def find_selected_revision(self, file_path, current_line):
        output = self._exec(
            'git blame -L %d,%d %s' % (current_line, current_line, file_path))
        return re.match(r'^(\w+)', output).group(1)

    def get_default_branch(self):
        remote = self.find_bitbucket_remote()
        return self._exec('git rev-parse --abbrev-ref refs/remotes/%s/HEAD'
                          % remote).strip()

    def get_pull_request_id(self, target_revision):
        default_branch = self.get_default_branch()
        revspec = '%s..%s' % (target_revision, default_branch)

        # First find the merge commit where the given commit was merged into
        # the default branch.
        ancestry_path = self._exec('git rev-list %s --ancestry-path' %
                                   revspec).splitlines()
        first_parent = self._exec('git rev-list %s --first-parent' %
                                  revspec).splitlines()
        common = set(ancestry_path) & set(first_parent)
        merge_revision = next(rev for rev in reversed(ancestry_path)
                              if rev in common)

        # Look at the commit message for the merge for a reference to the PR.
        info = self._exec('git show --oneline %s' % merge_revision)
        pull_request_match = re.search(r'pull request #(\d+)', info)
        if pull_request_match:
            return int(pull_request_match.group(1))

        raise SublimeBucketError('Unable to determine the pull request where '
                                 'the commit was merged')

class MercurialBackend(BackendBase):
    command = 'hg'

    def get_remote_list(self):
        return self._exec('hg paths').splitlines()

    def find_current_revision(self):
        return self._exec('hg id -i').strip()

    def find_selected_revision(self, file_path, current_line):
        output = self._exec(
            'hg annotate -c %s | head -n %d | tail -n 1' % (file_path,
                                                            current_line))
        return re.match(r'^(\w+)', output).group(1)

    def get_default_branch(self):
        return 'default'

    def get_pull_request_id(self, target_revision):
        log_output = self._exec(
            'hg log -r "first(descendants(%s) and desc(\'pull request\') '
            'and merge())"' % target_revision)
        pull_request_match = re.search(r'pull request #(\d+)', log_output)
        if pull_request_match:
            return int(pull_request_match.group(1))

        raise SublimeBucketError('Unable to determine the pull request where '
                                 'the changeset was merged')

class SublimeBucketError(Exception):
    pass
<|endoftext|>


<commit_before>"""This module provides the SublimeLinter plugin class and supporting methods."""

from collections import defaultdict, deque
from functools import partial
import logging
import os
import time
import threading

import sublime
import sublime_plugin

from . import log_handler
from .lint import backend
from .lint import events
from .lint import queue
from .lint import persist, util, style
from .lint import reloader

logger = logging.getLogger(__name__)

def backup_old_settings():
    """
    Backup old settings.

    If user settings file in old format exists it is renamed to disable it
    and back it up.
    A message will be displayed to the user.
    """
    usr_dir_abs = os.path.join(sublime.packages_path(), "User")
    settings_file = os.path.join(usr_dir_abs, "SublimeLinter.sublime-settings")
    if os.path.exists(settings_file):
        path = "Packages/User/SublimeLinter.sublime-settings"
        settings = sublime.decode_value(sublime.load_resource(path))

        if "user" in settings:
            new_name = "SublimeLinter (old).sublime-settings"
            new_path = os.path.join(usr_dir_abs, new_name)
            os.rename(settings_file, new_path)
            msg = "SublimeLinter\n\nYour settings have been backed up to:\n{}\nin Packages/User/".format(new_name)  # noqa: 501
            sublime.message_dialog(msg)

def plugin_loaded():
    log_handler.install()
    backup_old_settings()

    try:
        from package_control import events
        if events.install('SublimeLinter') or events.post_upgrade('SublimeLinter'):
            # In case the user has an old version installed without the below
            # `unload`, we 'unload' here.
            persist.kill_switch = True
            persist.linter_classes.clear()

            sublime.set_timeout_async(show_restart_message, 100)
            return
    except ImportError:
        pass

    persist.api_ready = True
    persist.kill_switch = False
    persist.settings.load()
    logger.info("debug mode: on")
    logger.info("version: " + util.get_sl_version())
    style.read_gutter_theme()
    style.StyleParser()()

    # Lint the visible views from the active window on startup
    if persist.settings.get("lint_mode") in ("background", "load_save"):
        for view in visible_views():
            hit(view)

def plugin_unloaded():
    try:
        from package_control import events

        if events.pre_upgrade('SublimeLinter') or events.remove('SublimeLinter'):
            logger.info("Enable kill_switch.")
            persist.kill_switch = True
            persist.linter_classes.clear()
    except ImportError:
        pass

    queue.unload()
    persist.settings.unobserve()

class SublimeLinterReloadCommand(sublime_plugin.WindowCommand):
    def run(self):
        reloader.reload_everything()

def show_restart_message():
    window = sublime.active_window()
    window.run_command("sublime_linter_display_panel", {
        'msg': (
            'SublimeLinter has been installed or upgraded. '
            'Please restart Sublime Text.'
        )
    })

def visible_views():
    """Yield all visible views of the active window."""
    window = sublime.active_window()

    # Priority for the active view
    active_view = window.active_view()
    yield active_view

    num_groups = window.num_groups()
    for group_id in range(num_groups):
        view = window.active_view_in_group(group_id)
        if view != active_view:
            yield view

global_lock = threading.RLock()
guard_check_linters_for_view = defaultdict(threading.Lock)
buffer_syntaxes = {}

class BackendController(sublime_plugin.EventListener):
    def on_modified_async(self, view):
        if persist.settings.get('lint_mode') != 'background':
            return

        if not util.is_lintable(view):
            return

        hit(view)

    def on_activated_async(self, view):
        # If the user changes the buffers syntax via the command palette,
        # we get an 'activated' event right after. Since, it is very likely
        # that the linters change as well, we 'hit' immediately for users
        # convenience.
        # We also use this instead of the `on_load_async` event as 'load'
        # event, bc 'on_load' fires for preview buffers which is way too
        # early. This fires a bit too often for 'load_save' mode but it is
        # good enough.

        if persist.settings.get('lint_mode') in ('manual', 'save'):
            return

        if not util.is_lintable(view):
            return

        if has_syntax_changed(view):
            hit(view)

    def on_post_save_async(self, view):
        if persist.settings.get('lint_mode') == 'manual':
            return

        # check if the project settings changed
        if view.window() and view.window().project_file_name() == view.file_name():
            lint_all_views()
            return

        if not util.is_lintable(view):
            return

        hit(view, reason='on_save')

    def on_pre_close(self, view):
        bid = view.buffer_id()
        buffers = []
        for w in sublime.windows():
            for v in w.views():
                buffers.append(v.buffer_id())

        # Cleanup bid-based stores if this is the last view on the buffer
        if buffers.count(bid) <= 1:
            persist.errors.pop(bid, None)
            persist.view_linters.pop(bid, None)

            guard_check_linters_for_view.pop(bid, None)
            buffer_syntaxes.pop(bid, None)
            queue.cleanup(bid)

def has_syntax_changed(view):
    bid = view.buffer_id()
    current_syntax = util.get_syntax(view)

    try:
        old_value = buffer_syntaxes[bid]
    except KeyError:
        return True
    else:
        return old_value != current_syntax
    finally:
        buffer_syntaxes[bid] = current_syntax

class SublimeLinterLintCommand(sublime_plugin.TextCommand):
    """A command that lints the current view if it has a linter."""

    def is_enabled(self):
        """
        Return True if the current view can be linted.

        If the view has *only* file-only linters, it can be linted
        only if the view is not dirty.

        Otherwise it can be linted.
        """
        has_non_file_only_linter = False

        bid = self.view.buffer_id()
        linters = persist.view_linters.get(bid, [])

        for lint in linters:
            if lint.tempfile_suffix != '-':
                has_non_file_only_linter = True
                break

        if not has_non_file_only_linter:
            return not self.view.is_dirty()

        return True

    def run(self, edit):
        """Lint the current view."""
        hit(self.view, reason='on_user_request')

class SublimeLinterConfigChanged(sublime_plugin.ApplicationCommand):
    def run(self):
        lint_all_views()

def lint_all_views():
    """Mimic a modification of all views, which will trigger a relint."""
    for window in sublime.windows():
        for view in window.views():
            if view.buffer_id() in persist.view_linters:
                hit(view)

def hit(view, reason=None):
    """Record an activity that could trigger a lint and enqueue a desire to lint."""
    bid = view.buffer_id()

    delay = get_delay(reason)
    logger.info('Delay buffer {} for {:.2}s'.format(bid, delay))
    lock = guard_check_linters_for_view[bid]
    view_has_changed = make_view_has_changed_fn(view)
    fn = partial(lint, view, view_has_changed, lock)
    queue.debounce(fn, delay=delay, key=bid)

def lint(view, view_has_changed, lock):
    """Lint the view with the given id.

    This method is called asynchronously by queue.Daemon when a lint
    request is pulled off the queue.
    """
    # We call `get_linters_for_view` first and unconditionally for its
    # side-effect. Yeah, it's a *getter* LOL.
    with lock:  # We're already debounced, so races are actually unlikely.
        linters = get_linters_for_view(view)

    linters = [
        linter for linter in linters
        if not skip_linter(linter, view)]
    if not linters:
        return

    # Very, very unlikely that `view_has_changed` is already True at this
    # point, but it also implements the kill_switch, so we ask here
    if view_has_changed():  # abort early
        return

    if not environment_is_ready():
        hit(view)
        return

    bid = view.buffer_id()

    events.broadcast(events.LINT_START, {'buffer_id': bid})
    start_time = time.time()

    next = partial(update_buffer_errors, bid, view_has_changed)
    backend.lint_view(linters, view, view_has_changed, next)

    end_time = time.time()
    runtime = end_time - start_time
    logger.info('Linting buffer {} took {:.2f}s'.format(bid, runtime))
    remember_runtime(runtime)
    events.broadcast(events.LINT_END, {'buffer_id': bid})

def skip_linter(linter, view):
    # A 'saved-file-only' linter does not run on unsaved views
    if linter.tempfile_suffix == '-' and view.is_dirty():
        return True

    return False

def update_buffer_errors(bid, view_has_changed, linter, errors):
    """Persist lint error changes and broadcast."""
    if view_has_changed():  # abort early
        return

    all_errors = [error for error in persist.errors[bid]
                  if error['linter'] != linter.name] + errors
    persist.errors[bid] = all_errors

    events.broadcast(events.LINT_RESULT, {
        'buffer_id': bid,
        'linter_name': linter.name,
        'errors': errors
    })

def get_linters_for_view(view):
    """Check and eventually instantiate linters for a view."""
    bid = view.buffer_id()

    linters = persist.view_linters.get(bid, set())
    wanted_linter_classes = {
        linter_class
        for linter_class in persist.linter_classes.values()
        if linter_class.can_lint_view(view)
    }
    current_linter_classes = {linter.__class__ for linter in linters}

    window = view.window()
    if window:
        window.run_command('sublime_linter_assigned', {
            'bid': bid,
            'linter_names': [
                linter_class.name for linter_class in wanted_linter_classes]
        })

    if current_linter_classes != wanted_linter_classes:
        unchanged_buffer = lambda: False  # noqa: E731
        for linter in (current_linter_classes - wanted_linter_classes):
            update_buffer_errors(bid, unchanged_buffer, linter, [])

        syntax = util.get_syntax(view)
        logger.info("detected syntax: {}".format(syntax))

        linters = {
            linter_class(view, syntax)
            for linter_class in wanted_linter_classes
        }
        persist.view_linters[bid] = linters

    return linters

def make_view_has_changed_fn(view):
    initial_change_count = view.change_count()

    def view_has_changed():
        if persist.kill_switch:
            logger.warning(
                'Aborting lint. SublimeLinter needs a restart of Sublime.')
            return True

        changed = view.change_count() != initial_change_count
        if changed:
            logger.info(
                'Buffer {} inconsistent. Aborting lint.'
                .format(view.buffer_id()))

        return changed

    return view_has_changed

ENV_READY_TIMEOUT = 60
ENV_IS_READY_STATE = {
    'ready': False if sublime.platform() == 'osx' else True,
    'start_time': None,
    'first_path': None
}

def environment_is_ready():
    ready = ENV_IS_READY_STATE['ready']
    if ready:
        return True

    path = os.environ['PATH']
    if '/local/' in path:
        logger.info('Env seems ok: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    start_time = ENV_IS_READY_STATE['start_time']
    if start_time is None:
        ENV_IS_READY_STATE['start_time'] = time.time()
    elif (time.time() - start_time) > ENV_READY_TIMEOUT:
        logger.warning('Env timeout: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    first_path = ENV_IS_READY_STATE['first_path']
    if first_path is None:
        logger.info('Env first path: PATH: {}'.format(path))
        logger.info(
            'Wait until env is loaded but max {}s.'.format(ENV_READY_TIMEOUT))
        ENV_IS_READY_STATE['first_path'] = path
    elif path != first_path:
        logger.info('Env path loaded: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    return False

elapsed_runtimes = deque([0.6] * 3, maxlen=10)
MIN_DEBOUNCE_DELAY = 0.05
MAX_AUTOMATIC_DELAY = 2.0

def get_delay(reason=None):
    """Return the delay between a lint request and when it will be processed."""
    if reason == 'on_user_request':
        return 0

    # Need to debounce on_save bc Sublime will fire the event per buffer
    if reason == 'on_save':
        return MIN_DEBOUNCE_DELAY

    runtimes = sorted(elapsed_runtimes)
    middle = runtimes[len(runtimes) // 2]
    return max(
        max(MIN_DEBOUNCE_DELAY, persist.settings.get('delay')),
        min(MAX_AUTOMATIC_DELAY, middle / 2)
    )

def remember_runtime(elapsed_runtime):
    with global_lock:
        elapsed_runtimes.append(elapsed_runtime)
<commit_msg>Fix comment<commit_after>"""This module provides the SublimeLinter plugin class and supporting methods."""

from collections import defaultdict, deque
from functools import partial
import logging
import os
import time
import threading

import sublime
import sublime_plugin

from . import log_handler
from .lint import backend
from .lint import events
from .lint import queue
from .lint import persist, util, style
from .lint import reloader

logger = logging.getLogger(__name__)

def backup_old_settings():
    """
    Backup old settings.

    If user settings file in old format exists it is renamed to disable it
    and back it up.
    A message will be displayed to the user.
    """
    usr_dir_abs = os.path.join(sublime.packages_path(), "User")
    settings_file = os.path.join(usr_dir_abs, "SublimeLinter.sublime-settings")
    if os.path.exists(settings_file):
        path = "Packages/User/SublimeLinter.sublime-settings"
        settings = sublime.decode_value(sublime.load_resource(path))

        if "user" in settings:
            new_name = "SublimeLinter (old).sublime-settings"
            new_path = os.path.join(usr_dir_abs, new_name)
            os.rename(settings_file, new_path)
            msg = "SublimeLinter\n\nYour settings have been backed up to:\n{}\nin Packages/User/".format(new_name)  # noqa: 501
            sublime.message_dialog(msg)

def plugin_loaded():
    log_handler.install()
    backup_old_settings()

    try:
        from package_control import events
        if events.install('SublimeLinter') or events.post_upgrade('SublimeLinter'):
            # In case the user has an old version installed without the below
            # `unload`, we 'unload' here.
            persist.kill_switch = True
            persist.linter_classes.clear()

            sublime.set_timeout_async(show_restart_message, 100)
            return
    except ImportError:
        pass

    persist.api_ready = True
    persist.kill_switch = False
    persist.settings.load()
    logger.info("debug mode: on")
    logger.info("version: " + util.get_sl_version())
    style.read_gutter_theme()
    style.StyleParser()()

    # Lint the visible views from the active window on startup
    if persist.settings.get("lint_mode") in ("background", "load_save"):
        for view in visible_views():
            hit(view)

def plugin_unloaded():
    try:
        from package_control import events

        if events.pre_upgrade('SublimeLinter') or events.remove('SublimeLinter'):
            logger.info("Enable kill_switch.")
            persist.kill_switch = True
            persist.linter_classes.clear()
    except ImportError:
        pass

    queue.unload()
    persist.settings.unobserve()

class SublimeLinterReloadCommand(sublime_plugin.WindowCommand):
    def run(self):
        reloader.reload_everything()

def show_restart_message():
    window = sublime.active_window()
    window.run_command("sublime_linter_display_panel", {
        'msg': (
            'SublimeLinter has been installed or upgraded. '
            'Please restart Sublime Text.'
        )
    })

def visible_views():
    """Yield all visible views of the active window."""
    window = sublime.active_window()

    # Priority for the active view
    active_view = window.active_view()
    yield active_view

    num_groups = window.num_groups()
    for group_id in range(num_groups):
        view = window.active_view_in_group(group_id)
        if view != active_view:
            yield view

global_lock = threading.RLock()
guard_check_linters_for_view = defaultdict(threading.Lock)
buffer_syntaxes = {}

class BackendController(sublime_plugin.EventListener):
    def on_modified_async(self, view):
        if persist.settings.get('lint_mode') != 'background':
            return

        if not util.is_lintable(view):
            return

        hit(view)

    def on_activated_async(self, view):
        # If the user changes the buffers syntax via the command palette,
        # we get an 'activated' event right after. Since, it is very likely
        # that the linters change as well, we 'hit' immediately for users
        # convenience.
        # We also use this instead of the `on_load_async` event as 'load'
        # event, bc 'on_load' fires for preview buffers which is way too
        # early. This fires a bit too often for 'load_save' mode but it is
        # good enough.

        if persist.settings.get('lint_mode') in ('manual', 'save'):
            return

        if not util.is_lintable(view):
            return

        if has_syntax_changed(view):
            hit(view)

    def on_post_save_async(self, view):
        if persist.settings.get('lint_mode') == 'manual':
            return

        # check if the project settings changed
        if view.window() and view.window().project_file_name() == view.file_name():
            lint_all_views()
            return

        if not util.is_lintable(view):
            return

        hit(view, reason='on_save')

    def on_pre_close(self, view):
        bid = view.buffer_id()
        buffers = []
        for w in sublime.windows():
            for v in w.views():
                buffers.append(v.buffer_id())

        # Cleanup bid-based stores if this is the last view on the buffer
        if buffers.count(bid) <= 1:
            persist.errors.pop(bid, None)
            persist.view_linters.pop(bid, None)

            guard_check_linters_for_view.pop(bid, None)
            buffer_syntaxes.pop(bid, None)
            queue.cleanup(bid)

def has_syntax_changed(view):
    bid = view.buffer_id()
    current_syntax = util.get_syntax(view)

    try:
        old_value = buffer_syntaxes[bid]
    except KeyError:
        return True
    else:
        return old_value != current_syntax
    finally:
        buffer_syntaxes[bid] = current_syntax

class SublimeLinterLintCommand(sublime_plugin.TextCommand):
    """A command that lints the current view if it has a linter."""

    def is_enabled(self):
        """
        Return True if the current view can be linted.

        If the view has *only* file-only linters, it can be linted
        only if the view is not dirty.

        Otherwise it can be linted.
        """
        has_non_file_only_linter = False

        bid = self.view.buffer_id()
        linters = persist.view_linters.get(bid, [])

        for lint in linters:
            if lint.tempfile_suffix != '-':
                has_non_file_only_linter = True
                break

        if not has_non_file_only_linter:
            return not self.view.is_dirty()

        return True

    def run(self, edit):
        """Lint the current view."""
        hit(self.view, reason='on_user_request')

class SublimeLinterConfigChanged(sublime_plugin.ApplicationCommand):
    def run(self):
        lint_all_views()

def lint_all_views():
    """Mimic a modification of all views, which will trigger a relint."""
    for window in sublime.windows():
        for view in window.views():
            if view.buffer_id() in persist.view_linters:
                hit(view)

def hit(view, reason=None):
    """Record an activity that could trigger a lint and enqueue a desire to lint."""
    bid = view.buffer_id()

    delay = get_delay(reason)
    logger.info('Delay buffer {} for {:.2}s'.format(bid, delay))
    lock = guard_check_linters_for_view[bid]
    view_has_changed = make_view_has_changed_fn(view)
    fn = partial(lint, view, view_has_changed, lock)
    queue.debounce(fn, delay=delay, key=bid)

def lint(view, view_has_changed, lock):
    """Lint the view with the given id.

    This method is called asynchronously by queue.Daemon when a lint
    request is pulled off the queue.
    """
    # We call `get_linters_for_view` first and unconditionally for its
    # side-effect. Yeah, it's a *getter* LOL.
    with lock:  # We're already debounced, so races are actually unlikely.
        linters = get_linters_for_view(view)

    linters = [
        linter for linter in linters
        if not skip_linter(linter, view)]
    if not linters:
        return

    # Very, very unlikely that `view_has_changed` is already True at this
    # point, but it also implements the kill_switch, so we ask here
    if view_has_changed():  # abort early
        return

    if not environment_is_ready():
        hit(view)
        return

    bid = view.buffer_id()

    events.broadcast(events.LINT_START, {'buffer_id': bid})
    start_time = time.time()

    next = partial(update_buffer_errors, bid, view_has_changed)
    backend.lint_view(linters, view, view_has_changed, next)

    end_time = time.time()
    runtime = end_time - start_time
    logger.info('Linting buffer {} took {:.2f}s'.format(bid, runtime))
    remember_runtime(runtime)
    events.broadcast(events.LINT_END, {'buffer_id': bid})

def skip_linter(linter, view):
    # A 'saved-file-only' linter does not run on unsaved views
    if linter.tempfile_suffix == '-' and view.is_dirty():
        return True

    return False

def update_buffer_errors(bid, view_has_changed, linter, errors):
    """Persist lint error changes and broadcast."""
    if view_has_changed():  # abort early
        return

    all_errors = [error for error in persist.errors[bid]
                  if error['linter'] != linter.name] + errors
    persist.errors[bid] = all_errors

    events.broadcast(events.LINT_RESULT, {
        'buffer_id': bid,
        'linter_name': linter.name,
        'errors': errors
    })

def get_linters_for_view(view):
    """Check and eventually instantiate linters for a view."""
    bid = view.buffer_id()

    linters = persist.view_linters.get(bid, set())
    wanted_linter_classes = {
        linter_class
        for linter_class in persist.linter_classes.values()
        if linter_class.can_lint_view(view)
    }
    current_linter_classes = {linter.__class__ for linter in linters}

    window = view.window()
    if window:
        window.run_command('sublime_linter_assigned', {
            'bid': bid,
            'linter_names': [
                linter_class.name for linter_class in wanted_linter_classes]
        })

    if current_linter_classes != wanted_linter_classes:
        unchanged_buffer = lambda: False  # noqa: E731
        for linter in (current_linter_classes - wanted_linter_classes):
            update_buffer_errors(bid, unchanged_buffer, linter, [])

        syntax = util.get_syntax(view)
        logger.info("detected syntax: {}".format(syntax))

        linters = {
            linter_class(view, syntax)
            for linter_class in wanted_linter_classes
        }
        persist.view_linters[bid] = linters

    return linters

def make_view_has_changed_fn(view):
    initial_change_count = view.change_count()

    def view_has_changed():
        if persist.kill_switch:
            logger.warning(
                'Aborting lint. SublimeLinter needs a restart of Sublime.')
            return True

        changed = view.change_count() != initial_change_count
        if changed:
            logger.info(
                'Buffer {} inconsistent. Aborting lint.'
                .format(view.buffer_id()))

        return changed

    return view_has_changed

ENV_READY_TIMEOUT = 60
ENV_IS_READY_STATE = {
    'ready': False if sublime.platform() == 'osx' else True,
    'start_time': None,
    'first_path': None
}

def environment_is_ready():
    ready = ENV_IS_READY_STATE['ready']
    if ready:
        return True

    path = os.environ['PATH']
    if '/local/' in path:
        logger.info('Env seems ok: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    start_time = ENV_IS_READY_STATE['start_time']
    if start_time is None:
        ENV_IS_READY_STATE['start_time'] = time.time()
    elif (time.time() - start_time) > ENV_READY_TIMEOUT:
        logger.warning('Env timeout: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    first_path = ENV_IS_READY_STATE['first_path']
    if first_path is None:
        logger.info('Env first path: PATH: {}'.format(path))
        logger.info(
            'Wait until env is loaded but max {}s.'.format(ENV_READY_TIMEOUT))
        ENV_IS_READY_STATE['first_path'] = path
    elif path != first_path:
        logger.info('Env path loaded: PATH: {}'.format(path))
        ENV_IS_READY_STATE['ready'] = True
        return True

    return False

elapsed_runtimes = deque([0.6] * 3, maxlen=10)
MIN_DEBOUNCE_DELAY = 0.05
MAX_AUTOMATIC_DELAY = 2.0

def get_delay(reason=None):
    """Return the delay between a lint request and when it will be processed."""
    if reason == 'on_user_request':
        return 0

    # Need to debounce on_save bc Sublime will fire the event per view
    if reason == 'on_save':
        return MIN_DEBOUNCE_DELAY

    runtimes = sorted(elapsed_runtimes)
    middle = runtimes[len(runtimes) // 2]
    return max(
        max(MIN_DEBOUNCE_DELAY, persist.settings.get('delay')),
        min(MAX_AUTOMATIC_DELAY, middle / 2)
    )

def remember_runtime(elapsed_runtime):
    with global_lock:
        elapsed_runtimes.append(elapsed_runtime)
<|endoftext|>


<commit_before>import pytest

from plenum.common.metrics_collector import KvStoreMetricsFormat, MetricsName, TMP_METRIC
from plenum.test.helper import sdk_send_random_and_check, max_3pc_batch_limits
from storage.helper import initKeyValueStorage

@pytest.fixture(scope="module")
def tconf(tconf):
    with max_3pc_batch_limits(tconf, size=3) as tconf:
        old_type = tconf.METRICS_COLLECTOR_TYPE
        tconf.METRICS_COLLECTOR_TYPE = 'kv'
        yield tconf
        tconf.METRICS_COLLECTOR_TYPE = old_type

def test_kv_store_metrics_config(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_client):
    total_time = 1.5 * tconf.PerfCheckFreq
    total_iters = 5
    iter_time = total_time / total_iters

    for _ in range(total_iters):
        sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 15)
        looper.runFor(iter_time)

    for node in txnPoolNodeSet:
        storage = initKeyValueStorage(tconf.METRICS_KV_STORAGE,
                                      node.dataLocation,
                                      tconf.METRICS_KV_DB_NAME,
                                      read_only=True)

        events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]

        # Check that metrics are actually written
        assert len(events) > 0

        # Check that all events are stored in correct order
        assert sorted(events, key=lambda v: v.timestamp) == events

        # We don't expect some events in this test
        unexpected_events = {
            MetricsName.GC_UNCOLLECTABLE_OBJECTS,

            MetricsName.PROCESS_CHECKPOINT_TIME,
            MetricsName.SEND_CHECKPOINT_TIME,
            MetricsName.BACKUP_PROCESS_CHECKPOINT_TIME,
            MetricsName.BACKUP_SEND_CHECKPOINT_TIME,
            MetricsName.PROCESS_CONSISTENCY_PROOF_TIME,
            MetricsName.PROCESS_CATCHUP_REQ_TIME,
            MetricsName.PROCESS_CATCHUP_REP_TIME,
            MetricsName.NODE_CHECK_NODE_REQUEST_SPIKE,
            MetricsName.NODE_SEND_REJECT_TIME,

            # Obsolete metrics
            MetricsName.DESERIALIZE_DURING_UNPACK_TIME,

            # TODO: reduce monitor window so these events are also captured
            MetricsName.MONITOR_AVG_THROUGHPUT,
            MetricsName.BACKUP_MONITOR_AVG_THROUGHPUT,

            # Temporary metrics
            MetricsName.STORAGE_IDR_CACHE_READERS,
            MetricsName.STORAGE_IDR_CACHE_TABLES_NUM,
            MetricsName.STORAGE_IDR_CACHE_TABLES_SIZE,
            MetricsName.STORAGE_ATTRIBUTE_STORE_READERS,
            MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_NUM,
            MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_SIZE
        }

        # Don't expect some metrics from master primary
        if node.master_replica.isPrimary:
            unexpected_events.add(MetricsName.PROCESS_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.SEND_PREPARE_TIME)
        else:
            unexpected_events.add(MetricsName.SEND_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.CREATE_3PC_BATCH_TIME)
            unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)

        # Don't expect some metrics from backup primary
        assert node.replicas.num_replicas == 2
        if node.replicas[1].isPrimary:
            unexpected_events.add(MetricsName.BACKUP_PROCESS_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.BACKUP_SEND_PREPARE_TIME)
        else:
            unexpected_events.add(MetricsName.BACKUP_SEND_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.BACKUP_CREATE_3PC_BATCH_TIME)
            unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)

        if not node.primaryDecider:
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_ACTION_QUEUE)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_AQ_STASH)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_REPEATING_ACTIONS)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_SCHEDULED)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_INBOX)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_OUTBOX)

        # Check that all event types happened during test
        metric_names = {ev.name for ev in events}
        for t in MetricsName:
            if t in unexpected_events or t > TMP_METRIC:
                continue
            assert t in metric_names
<commit_msg>Fix rare flakiness of test_metrics_config<commit_after>import pytest

from plenum.common.metrics_collector import KvStoreMetricsFormat, MetricsName, TMP_METRIC
from plenum.test.helper import sdk_send_random_and_check, max_3pc_batch_limits
from storage.helper import initKeyValueStorage

@pytest.fixture(scope="module")
def tconf(tconf):
    with max_3pc_batch_limits(tconf, size=3) as tconf:
        old_type = tconf.METRICS_COLLECTOR_TYPE
        tconf.METRICS_COLLECTOR_TYPE = 'kv'
        yield tconf
        tconf.METRICS_COLLECTOR_TYPE = old_type

def test_kv_store_metrics_config(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_client):
    total_time = 1.5 * tconf.PerfCheckFreq
    total_iters = 5
    iter_time = total_time / total_iters

    for _ in range(total_iters):
        sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 15)
        looper.runFor(iter_time)

    for node in txnPoolNodeSet:
        storage = initKeyValueStorage(tconf.METRICS_KV_STORAGE,
                                      node.dataLocation,
                                      tconf.METRICS_KV_DB_NAME,
                                      read_only=True)

        events = [KvStoreMetricsFormat.decode(k, v) for k, v in storage.iterator()]

        # Check that metrics are actually written
        assert len(events) > 0

        # Check that all events are stored in correct order
        assert sorted(events, key=lambda v: v.timestamp) == events

        # We don't expect some events in this test
        unexpected_events = {
            MetricsName.GC_UNCOLLECTABLE_OBJECTS,
            MetricsName.GC_GEN2_COLLECTED_OBJECTS,

            MetricsName.PROCESS_CHECKPOINT_TIME,
            MetricsName.SEND_CHECKPOINT_TIME,
            MetricsName.BACKUP_PROCESS_CHECKPOINT_TIME,
            MetricsName.BACKUP_SEND_CHECKPOINT_TIME,
            MetricsName.PROCESS_CONSISTENCY_PROOF_TIME,
            MetricsName.PROCESS_CATCHUP_REQ_TIME,
            MetricsName.PROCESS_CATCHUP_REP_TIME,
            MetricsName.NODE_CHECK_NODE_REQUEST_SPIKE,
            MetricsName.NODE_SEND_REJECT_TIME,

            # Obsolete metrics
            MetricsName.DESERIALIZE_DURING_UNPACK_TIME,

            # TODO: reduce monitor window so these events are also captured
            MetricsName.MONITOR_AVG_THROUGHPUT,
            MetricsName.BACKUP_MONITOR_AVG_THROUGHPUT,

            # Temporary metrics
            MetricsName.STORAGE_IDR_CACHE_READERS,
            MetricsName.STORAGE_IDR_CACHE_TABLES_NUM,
            MetricsName.STORAGE_IDR_CACHE_TABLES_SIZE,
            MetricsName.STORAGE_ATTRIBUTE_STORE_READERS,
            MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_NUM,
            MetricsName.STORAGE_ATTRIBUTE_STORE_TABLES_SIZE
        }

        # Don't expect some metrics from master primary
        if node.master_replica.isPrimary:
            unexpected_events.add(MetricsName.PROCESS_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.SEND_PREPARE_TIME)
        else:
            unexpected_events.add(MetricsName.SEND_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.CREATE_3PC_BATCH_TIME)
            unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)

        # Don't expect some metrics from backup primary
        assert node.replicas.num_replicas == 2
        if node.replicas[1].isPrimary:
            unexpected_events.add(MetricsName.BACKUP_PROCESS_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.BACKUP_SEND_PREPARE_TIME)
        else:
            unexpected_events.add(MetricsName.BACKUP_SEND_PREPREPARE_TIME)
            unexpected_events.add(MetricsName.BACKUP_CREATE_3PC_BATCH_TIME)
            unexpected_events.add(MetricsName.BLS_UPDATE_PREPREPARE_TIME)

        if not node.primaryDecider:
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_ACTION_QUEUE)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_AQ_STASH)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_REPEATING_ACTIONS)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_SCHEDULED)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_INBOX)
            unexpected_events.add(MetricsName.PRIMARY_DECIDER_OUTBOX)

        # Check that all event types happened during test
        metric_names = {ev.name for ev in events}
        for t in MetricsName:
            if t in unexpected_events or t > TMP_METRIC:
                continue
            assert t in metric_names
<|endoftext|>


<commit_before>import json
from datetime import timedelta

from core import Feed
from core.observables import Hash, File
from core.config.config import yeti_config
import logging

class VirusTotalHunting(Feed):

    default_values = {
        "frequency": timedelta(minutes=5),
        "name": "VirusTotalHunting",
        "source": "https://www.virustotal.com/intelligence/hunting/",
        "description": "Feed of hunting of VirusTotal",

    }

    settings = {
        'vt_url_hunting': {
            'name': 'VT Url Hunting',
            'description': 'on Virus Total you can make a feed in json or xml in tab hunting'
        }
    }

    def update(self):
        api_key = yeti_config.get('vt', 'key')

        if api_key:
            self.source = 'https://www.virustotal.com/intelligence/hunting/notifications-feed/?key=%s' % api_key
            for item in self.update_json()['notifications']:
                self.analyze(item)
        else:
            logging.error("Your VT API key is not set in the confile file")

    def analyze(self, item):
        tags = []
        json_string = json.dumps(item)
        context = {'source': self.name}

        f_vt = File.get_or_create(value='FILE:{}'.format(item['sha256']))

        sha256 = Hash.get_or_create(value=item['sha256'])

        md5 = Hash.get_or_create(value=item['md5'])

        sha1 = Hash.get_or_create(value=item['sha1'])

        f_vt.active_link_to(md5, 'md5', self.name)
        f_vt.active_link_to(sha1, 'sha1', self.name)
        f_vt.active_link_to(sha256, 'sha256', self.name)

        tags.append(item['ruleset_name'])
        tags.append(item['type'])

        context['raw'] = json_string
        context['size'] = item['size']
        context['score vt'] = '%s/%s' % (item['positives'], item['total'])

        f_vt.tag(tags)
        f_vt.add_context(context)<commit_msg>adapt virustotal_hunting.py<commit_after>import logging
from datetime import timedelta

from core import Feed
from core.config.config import yeti_config
from core.observables import Hash, File

class VirusTotalHunting(Feed):

    default_values = {
        "frequency": timedelta(minutes=5),
        "name": "VirusTotalHunting",
        "source": "https://www.virustotal.com/intelligence/hunting/",
        "description": "Feed of hunting of VirusTotal",

    }

    settings = {
        'vt_url_hunting': {
            'name': 'VT Url Hunting',
            'description': 'on Virus Total you can make a feed in json or xml in tab hunting'
        }
    }

    def update(self):
        api_key = yeti_config.get('vt', 'key')

        if api_key:
            self.source = 'https://www.virustotal.com/intelligence/hunting/notifications-feed/?key=%s' % api_key
            for index,item in self.update_json(key='notifications'):
                self.analyze(item)
        else:
            logging.error("Your VT API key is not set in the confile file")

    def analyze(self, item):
        tags = []

        json_string = item.to_json()
        context = {'source': self.name}

        f_vt = File.get_or_create(value='FILE:{}'.format(item['sha256']))

        sha256 = Hash.get_or_create(value=item['sha256'])

        md5 = Hash.get_or_create(value=item['md5'])

        sha1 = Hash.get_or_create(value=item['sha1'])

        f_vt.active_link_to(md5, 'md5', self.name)
        f_vt.active_link_to(sha1, 'sha1', self.name)
        f_vt.active_link_to(sha256, 'sha256', self.name)

        tags.append(item['ruleset_name'])
        tags.append(item['type'])

        context['raw'] = json_string
        context['size'] = item['size']
        context['score vt'] = '%s/%s' % (item['positives'], item['total'])

        f_vt.tag(tags)
        f_vt.add_context(context)<|endoftext|>