<commit_before>import html5lib
from html5lib.constants import tokenTypes
from html5lib.sanitizer import HTMLSanitizerMixin
from html5lib.serializer.htmlserializer import HTMLSerializer
from html5lib.tokenizer import HTMLTokenizer

class StyleSanitizerMixin(HTMLSanitizerMixin):
    def sanitize_token(self, token):
        if not hasattr(self, 'in_style_tag'):
            self.in_style_tag = False

        if token['type'] in (tokenTypes['StartTag'], tokenTypes['EndTag'],
                             tokenTypes['EmptyTag']):
            if token['name'] == 'style':
                if token['type'] == tokenTypes['StartTag']:
                    self.in_style_tag = True
                    return
                if token['type'] == tokenTypes['EndTag']:
                    self.in_style_tag = False
                    return

        if self.in_style_tag:
            # We're in a style tag, so we want to discard this token
            return

        # We're not in a style tag, so we want to keep this token

        return token

class StyleSanitizer(HTMLTokenizer, StyleSanitizerMixin):
    def __iter__(self):
        for token in HTMLTokenizer.__iter__(self):
            token = self.sanitize_token(token)
            if token:
                yield token

def strip_styles(text):
    parser = html5lib.HTMLParser(tokenizer=StyleSanitizer)
    domtree = parser.parseFragment(text)
    walker = html5lib.treewalkers.getTreeWalker('simpletree')
    stream = walker(domtree)
    serializer = HTMLSerializer(quote_attr_values=True,
                                omit_optional_tags=False)
    return serializer.render(stream)
    <commit_msg>Fixing whitespace problem.<commit_after>import html5lib
from html5lib.constants import tokenTypes
from html5lib.sanitizer import HTMLSanitizerMixin
from html5lib.serializer.htmlserializer import HTMLSerializer
from html5lib.tokenizer import HTMLTokenizer

class StyleSanitizerMixin(HTMLSanitizerMixin):
    def sanitize_token(self, token):
        if not hasattr(self, 'in_style_tag'):
            self.in_style_tag = False

        if token['type'] in (tokenTypes['StartTag'], tokenTypes['EndTag'],
                             tokenTypes['EmptyTag']):
            if token['name'] == 'style':
                if token['type'] == tokenTypes['StartTag']:
                    self.in_style_tag = True
                    return
                if token['type'] == tokenTypes['EndTag']:
                    self.in_style_tag = False
                    return

        if self.in_style_tag:
            # We're in a style tag, so we want to discard this token
            return

        # We're not in a style tag, so we want to keep this token

        return token

class StyleSanitizer(HTMLTokenizer, StyleSanitizerMixin):
    def __iter__(self):
        for token in HTMLTokenizer.__iter__(self):
            token = self.sanitize_token(token)
            if token:
                yield token

def strip_styles(text):
    parser = html5lib.HTMLParser(tokenizer=StyleSanitizer)
    domtree = parser.parseFragment(text)
    walker = html5lib.treewalkers.getTreeWalker('simpletree')
    stream = walker(domtree)
    serializer = HTMLSerializer(quote_attr_values=True,
                                omit_optional_tags=False)
    return serializer.render(stream)
<|endoftext|>


<commit_before>#########################################################################
#                                                                       #
# Copyright (C) 2009  Sharoon Thomas                                    #
# Copyright (C) 2011 Akretion Sébastien BEAU sebastien.beau@akretion.com#
#                                                                       #
#This program is free software: you can redistribute it and/or modify   #
#it under the terms of the GNU General Public License as published by   #
#the Free Software Foundation, either version 3 of the License, or      #
#(at your option) any later version.                                    #
#                                                                       #
#This program is distributed in the hope that it will be useful,        #
#but WITHOUT ANY WARRANTY; without even the implied warranty of         #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          #
#GNU General Public License for more details.                           #
#                                                                       #
#You should have received a copy of the GNU General Public License      #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.  #
#########################################################################
{
    "name" : "Magento e-commerce",
    "version" : "6.1.2",
    "depends" : ["base",
                 "product",
                 "product_m2mcategories",
                 'delivery',
                 "base_sale_multichannels",
                 "product_images_olbs",
                 "product_links_sync",
                ],
    "author" : "MagentoERPconnect Core Editors",
    "description": """Magento E-commerce management
""",
    'images': [
        'images/main_menu.png',
        'images/instance.png',
        'images/sale_shop.png',
        'images/product.png',
        'images/magentocoreeditors.png',
        'images/magentoerpconnect.png',
    ],
    "website" : "https://launchpad.net/magentoerpconnect",
    "category" : "Generic Modules",
    "init_xml" : ['settings/magerp.product_category_attribute_options.csv',],
    "demo_xml" : [],
    "update_xml" : [
            'security/ir.model.access.csv',
            'settings/magerp.product_category_attribute_options.csv',
            'settings/external.referential.type.csv',
            'settings/1.5.0.0/external.referential.version.csv',
            'settings/1.5.0.0/external.mapping.template.csv',
            'settings/1.5.0.0/external.shop.group/external.mappinglines.template.csv',
            'settings/1.5.0.0/magerp.storeviews/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.shop/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner.category/external.mappinglines.template.csv',
            'settings/1.5.0.0/product.category/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut.set/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut.group/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut/external.mappinglines.template.csv',
            'settings/1.5.0.0/product/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.order/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.order.line/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner.address/external.mappinglines.template.csv',
            'settings/1.5.0.0/account.invoice/external.mappinglines.template.csv',
            'settings/1.5.0.0/product.link/external.mappinglines.template.csv',
            'settings/magerp_product_product_type.xml',
            'magerp_data.xml',
            'magerp_core_view.xml',
            'product_view.xml',
            'partner_view.xml',
            'sale_view.xml',
            'product_images_view.xml',
            'magerp_menu.xml',
            'delivery_view.xml',
            'product_links_view.xml',
            'wizard/open_product_by_attribut_set.xml',
                    ],
    "auto_install": False,
    "installable": True,
    'application': True,
}
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[MIGR] updated manifest<commit_after>#########################################################################
#                                                                       #
# Copyright (C) 2009  Sharoon Thomas                                    #
# Copyright (C) 2011 Akretion Sébastien BEAU sebastien.beau@akretion.com#
# Copyright 2011-2013 Camptocamp SA                                     #
#                                                                       #
#This program is free software: you can redistribute it and/or modify   #
#it under the terms of the GNU General Public License as published by   #
#the Free Software Foundation, either version 3 of the License, or      #
#(at your option) any later version.                                    #
#                                                                       #
#This program is distributed in the hope that it will be useful,        #
#but WITHOUT ANY WARRANTY; without even the implied warranty of         #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          #
#GNU General Public License for more details.                           #
#                                                                       #
#You should have received a copy of the GNU General Public License      #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.  #
#########################################################################
{
    "name" : "Magento Connector New Generation",
    "version" : "7.0",
    "depends" : [
                 "product",
                 'delivery',
                 "base_sale_multichannels",
                 "product_m2mcategories",
                 "product_images",
                 "product_links",
            ],
    "author" : "MagentoERPconnect Core Editors",
    "description": """
Magento E-commerce management
=============================

TODO

""",
    'images': [
        'images/main_menu.png',
        'images/instance.png',
        'images/sale_shop.png',
        'images/product.png',
        'images/magentocoreeditors.png',
        'images/magentoerpconnect.png',
    ],
    "website" : "https://launchpad.net/magentoerpconnect",
    "category" : "Generic Modules",
    "init_xml" : ['settings/magerp.product_category_attribute_options.csv',],
    "demo_xml" : [],
    "update_xml" : [
            'security/ir.model.access.csv',
            'settings/magerp.product_category_attribute_options.csv',
            'settings/external.referential.type.csv',
            'settings/1.5.0.0/external.referential.version.csv',
            'settings/1.5.0.0/external.mapping.template.csv',
            'settings/1.5.0.0/external.shop.group/external.mappinglines.template.csv',
            'settings/1.5.0.0/magerp.storeviews/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.shop/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner.category/external.mappinglines.template.csv',
            'settings/1.5.0.0/product.category/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut.set/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut.group/external.mappinglines.template.csv',
            'settings/1.5.0.0/attribut/external.mappinglines.template.csv',
            'settings/1.5.0.0/product/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.order/external.mappinglines.template.csv',
            'settings/1.5.0.0/sale.order.line/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner/external.mappinglines.template.csv',
            'settings/1.5.0.0/res.partner.address/external.mappinglines.template.csv',
            'settings/1.5.0.0/account.invoice/external.mappinglines.template.csv',
            'settings/1.5.0.0/product.link/external.mappinglines.template.csv',
            'settings/magerp_product_product_type.xml',
            'magerp_data.xml',
            'magerp_core_view.xml',
            'product_view.xml',
            'partner_view.xml',
            'sale_view.xml',
            'product_images_view.xml',
            'magerp_menu.xml',
            'delivery_view.xml',
            'product_links_view.xml',
            'wizard/open_product_by_attribut_set.xml',
            ],
    "auto_install": False,
    "installable": True,
    'application': True,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#    Copyright 2013 Akretion
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{'name': 'Magento Connector',
 'version': '2.5.0',
 'category': 'Connector',
 'depends': ['account',
             'product',
             'delivery',
             'sale_stock',
             'connector_ecommerce',
             'product_m2mcategories',
             ],
 'external_dependencies': {
     'python': ['magento'],
 },
 'author': "Connector Core Editors,Odoo Community Association (OCA)",
 'license': 'AGPL-3',
 'website': 'http://www.odoo-magento-connector.com',
 'description': """
Magento Connector
=================

This is the new release of the Open-Source connector linking OpenERP and
Magento also known under the name of **Magentoerpconnect**.  It is
build on top of the `connector`_ framework. It is is structured so that
it can be extended or modified easily from separate addons, a factor of
success when the implementations of Magento vary a lot.

Magento OpenERP Connector is mainly developed by the Connector Core
Editors, these being Camptocamp_ and Akretion_. The `source is on
GitHub`_.

This connector is designed to have a strong and efficient core, with the
ability to extend it with extension modules or local customizations.

In other words, the core module contains the minimal scope to run your
e-commerce with OpenERP and Magento. More advanced features are
installable using extensions.

It features:

Synchronizations:

* Import the partners and addresses book
* Import the customer groups (becomes partner tags)
* Import the categories of products, with translations
* Import the products, with translations and main image
* Import the sales orders
* Update the the stock quantities,
  with configuration of the warehouse and stock field to use
* Export the delivery orders
* Export the tracking numbers
* Create the invoices on Magento and get their number back
* Import the dependencies when they are not yet imported
  (ie. customer, products for sale order)

Automatizations:

* Use the `Automatic workflows` to automatize the workflow of the sales
  according to the payment method (confirm orders, create and reconcile
  payments, ...)
* Per payment method, choose when the orders are imported
  (only when a payment is received, always, never)
* Use the `Sales Exceptions` to prevents the processing of sales orders
  with issues

Technical points:

* Built on top of the `connector`_ framework
* Use the `connector_ecommerce`_ addon to share the e-commerce capabilities
  with other e-commerce addons
* Use the jobs system of the `connector`_ framework
* Create `connector`_ checkpoints when new records to verify are imported
* Support Magento 1.7, the support of earlier versions should be easy to
  add, the `connector`_ framework being designed to handle multiple
  versions with ease.
* Licensed under AGPL version 3
* Designed to be usable with multiple Magento or any other e-commerce backends
  in the same time.

Available extensions:

* Pricing
        allows to manage the prices in OpenERP using pricelists,
        prices are update to Magento when changed
* Export of partners (Experimental)
        Export new partners on Magento, not complete.

.. _connector: https://github.com/OCA/connector
.. _connector_ecommerce: https://github.com/OCA/connector-ecommerce
.. _Camptocamp: http://www.camptocamp.com
.. _Akretion: http://www.akretion.com
.. _`source is on GitHub`: https://github.com/OCA/connector-magento

""",
 'images': ['images/magento_backend.png',
            'images/jobs.png',
            'images/product_binding.png',
            'images/invoice_binding.png',
            'images/magentoerpconnect.png',
            ],
 'demo': [],
 'data': ['setting_view.xml',
          'magentoerpconnect_data.xml',
          'magento_model_view.xml',
          'product_view.xml',
          'partner_view.xml',
          'sale_view.xml',
          'invoice_view.xml',
          'magentoerpconnect_menu.xml',
          'delivery_view.xml',
          'stock_view.xml',
          'security/ir.model.access.csv',
          'payment_invoice.xml',
          ],
 'installable': False,
 'application': True,
 }
<commit_msg>Set magentoerpconnect to installable<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#    Copyright 2013 Akretion
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{'name': 'Magento Connector',
 'version': '2.5.0',
 'category': 'Connector',
 'depends': ['account',
             'product',
             'delivery',
             'sale_stock',
             'connector_ecommerce',
             'product_m2mcategories',
             ],
 'external_dependencies': {
     'python': ['magento'],
 },
 'author': "Connector Core Editors,Odoo Community Association (OCA)",
 'license': 'AGPL-3',
 'website': 'http://www.odoo-magento-connector.com',
 'description': """
Magento Connector
=================

This is the new release of the Open-Source connector linking OpenERP and
Magento also known under the name of **Magentoerpconnect**.  It is
build on top of the `connector`_ framework. It is is structured so that
it can be extended or modified easily from separate addons, a factor of
success when the implementations of Magento vary a lot.

Magento OpenERP Connector is mainly developed by the Connector Core
Editors, these being Camptocamp_ and Akretion_. The `source is on
GitHub`_.

This connector is designed to have a strong and efficient core, with the
ability to extend it with extension modules or local customizations.

In other words, the core module contains the minimal scope to run your
e-commerce with OpenERP and Magento. More advanced features are
installable using extensions.

It features:

Synchronizations:

* Import the partners and addresses book
* Import the customer groups (becomes partner tags)
* Import the categories of products, with translations
* Import the products, with translations and main image
* Import the sales orders
* Update the the stock quantities,
  with configuration of the warehouse and stock field to use
* Export the delivery orders
* Export the tracking numbers
* Create the invoices on Magento and get their number back
* Import the dependencies when they are not yet imported
  (ie. customer, products for sale order)

Automatizations:

* Use the `Automatic workflows` to automatize the workflow of the sales
  according to the payment method (confirm orders, create and reconcile
  payments, ...)
* Per payment method, choose when the orders are imported
  (only when a payment is received, always, never)
* Use the `Sales Exceptions` to prevents the processing of sales orders
  with issues

Technical points:

* Built on top of the `connector`_ framework
* Use the `connector_ecommerce`_ addon to share the e-commerce capabilities
  with other e-commerce addons
* Use the jobs system of the `connector`_ framework
* Create `connector`_ checkpoints when new records to verify are imported
* Support Magento 1.7, the support of earlier versions should be easy to
  add, the `connector`_ framework being designed to handle multiple
  versions with ease.
* Licensed under AGPL version 3
* Designed to be usable with multiple Magento or any other e-commerce backends
  in the same time.

Available extensions:

* Pricing
        allows to manage the prices in OpenERP using pricelists,
        prices are update to Magento when changed
* Export of partners (Experimental)
        Export new partners on Magento, not complete.

.. _connector: https://github.com/OCA/connector
.. _connector_ecommerce: https://github.com/OCA/connector-ecommerce
.. _Camptocamp: http://www.camptocamp.com
.. _Akretion: http://www.akretion.com
.. _`source is on GitHub`: https://github.com/OCA/connector-magento

""",
 'images': ['images/magento_backend.png',
            'images/jobs.png',
            'images/product_binding.png',
            'images/invoice_binding.png',
            'images/magentoerpconnect.png',
            ],
 'demo': [],
 'data': ['setting_view.xml',
          'magentoerpconnect_data.xml',
          'magento_model_view.xml',
          'product_view.xml',
          'partner_view.xml',
          'sale_view.xml',
          'invoice_view.xml',
          'magentoerpconnect_menu.xml',
          'delivery_view.xml',
          'stock_view.xml',
          'security/ir.model.access.csv',
          'payment_invoice.xml',
          ],
 'installable': True,
 'application': True,
 }
<|endoftext|>


<commit_before>"""
Sychronize local and remote resources.

To implement a new service, see the _SyncServiceInterface class.
"""

import abc
import copy
import datetime
from collections import OrderedDict

import requests
import threddsclient
from sqlalchemy import create_engine
from sqlalchemy.orm import Session

from magpie import db, models

def merge_local_and_remote_resources(resources_local, service_type, session):
    """Main function to sync resources with remote server"""
    remote_resources = _query_remote_resources_in_database(service_type, session=session)
    merged_resources = _merge_resources(resources_local, remote_resources)
    _sort_resources(merged_resources)
    return merged_resources

def _merge_resources(resources_local, resources_remote):
    """
    Merge resources_local and resources_remote, adding the following keys to the output:

        - remote_path: '/' separated string representing the remote path of the resource
        - matches_remote: True or False depending if the resource is present on the remote server
        - id: set to the value of 'remote_path' if the resource if remote only

    returns a dictionary of the form validated by 'is_valid_resource_schema'

    """
    if not resources_remote:
        return resources_local

    assert _is_valid_resource_schema(resources_local, ignore_resource_type=True)
    assert _is_valid_resource_schema(resources_remote)

    if not resources_local:
        raise ValueError("The resources must contain at least the service name.")

    # The first item is the service name. It is skipped so that only the resources are compared.
    service_name = resources_local.keys()[0]
    _, remote_values = resources_remote.popitem()
    resources_remote = {service_name: remote_values}

    # don't overwrite the input arguments
    merged_resources = copy.deepcopy(resources_local)

    def recurse(_resources_local, _resources_remote, remote_path="", remote_type_path=""):
        # loop local resources, looking for matches in remote resources
        for resource_name_local, values in _resources_local.items():
            current_path = "/".join([remote_path, str(resource_name_local)])

            matches_remote = resource_name_local in _resources_remote
            resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
            current_type_path = "/".join([remote_type_path, resource_type])

            values["remote_path"] = "" if matches_remote else current_path
            values["remote_type_path"] = current_type_path
            values["matches_remote"] = matches_remote
            values["resource_type"] = resource_type

            resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}

            recurse(values['children'], resource_remote_children, current_path, current_type_path)

        # loop remote resources, looking for matches in local resources
        for resource_name_remote, values in _resources_remote.items():
            if resource_name_remote not in _resources_local:
                current_path = "/".join([remote_path, str(resource_name_remote)])
                current_type_path = "/".join([remote_type_path, values['resource_type']])
                new_resource = {'permission_names': [],
                                'children': {},
                                'resource_type': values['resource_type'],
                                'id': current_path,
                                'remote_path': current_path,
                                'remote_type_path': current_type_path,
                                'matches_remote': True}
                _resources_local[resource_name_remote] = new_resource
                recurse(new_resource['children'], values['children'], current_path, current_type_path)

    recurse(merged_resources, resources_remote)

    assert _is_valid_resource_schema(merged_resources)

    return merged_resources

def _is_valid_resource_schema(resources, ignore_resource_type=False):
    """
    Returns True if the structure of the input dictionary is a tree of the form:

    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
                                      'resource_name_4': {'children': {}, 'resource_type': ...}
                                      },
                         'resource_type': ...
                         },
     'resource_name_2': {'children': {}, resource_type': ...}
     }
    :return: bool
    """
    for resource_name, values in resources.items():
        if 'children' not in values:
            return False
        if not ignore_resource_type and 'resource_type' not in values:
            return False
        if not isinstance(values['children'], (OrderedDict, dict)):
            return False
        return _is_valid_resource_schema(values['children'],
                                         ignore_resource_type=ignore_resource_type)
    return True

def _sort_resources(resources):
    """
    Sorts a resource dictionary of the type validated by '_is_valid_resource_schema'
    by using an OrderedDict
    :return: None
    """
    for resource_name, values in resources.items():
        values['children'] = OrderedDict(sorted(values['children'].iteritems()))
        return _sort_resources(values['children'])

class _SyncServiceInterface:
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def get_resources(self):
        """
        This is the function actually fetching the data from the remote service.
        Implement this for every specific service.

        :return: The returned dictionary must be validated by 'is_valid_resource_schema'
        """
        pass

class _SyncServiceGeoserver:
    def __init__(self, geoserver_url):
        self.geoserver_url = geoserver_url

    def get_resources(self):
        # Only workspaces are fetched for now
        resource_type = "route"
        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
        resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
        resp.raise_for_status()
        workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})

        workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}

        resources = {"geoserver-api": {"children": workspaces,
                                       "resource_type": resource_type}}
        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
        return resources

class _SyncServiceThreads(_SyncServiceInterface):
    DEPTH_DEFAULT = 2

    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
        self.thredds_url = thredds_url
        self.depth = depth
        self.kwargs = kwargs  # kwargs is passed to the requests.get method.

    def get_resources(self):
        def thredds_get_resources(url, depth, **kwargs):
            cat = threddsclient.read_url(url, **kwargs)
            name = cat.name
            resource_type = 'directory'
            if cat.datasets and cat.datasets[0].content_type != "application/directory":
                resource_type = 'file'

            tree_item = {name: {'children': {}, 'resource_type': resource_type}}

            if depth > 0:
                for reference in cat.flat_references():
                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))

            return tree_item

        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
        assert _is_valid_resource_schema(resources), 'Error in Interface implementation'
        return resources

class _SyncServiceDefault(_SyncServiceInterface):
    def __init__(self, _):
        pass

    def get_resources(self):
        return {}

SYNC_SERVICES = {
    "thredds": _SyncServiceThreads,
    "geoserver-api": _SyncServiceGeoserver,
}

def _ensure_sync_info_exists(service_resource_id, session):
    """
    Make sure the RemoteResourcesSyncInfo entry exists in the database.
    :param service_resource_id:
    :param session:
    """
    service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_resource_id, session)
    if not service_sync_info:
        sync_info = models.RemoteResourcesSyncInfo(service_id=service_resource_id)
        session.add(sync_info)
        session.flush()
        _create_main_resource(service_resource_id, session)

def _get_remote_resources(service, service_name):
    """
    Request rmeote resources, depending on service type.
    :param service:
    :param service_name:
    :return:
    """
    service_url = service.url
    if service_url.endswith("/"):  # remove trailing slash
        service_url = service_url[:-1]
    sync_service = SYNC_SERVICES.get(service_name.lower(), _SyncServiceDefault)(service_url)
    return sync_service.get_resources()

def _delete_records(service_id, session):
    """
    Delete all RemoteResource based on a Service.resource_id
    :param service_id:
    :param session:
    """
    session.query(models.RemoteResource).filter_by(service_id=service_id).delete()
    session.flush()

def _create_main_resource(service_id, session):
    """
    Creates a main resource for a service, whether one currently exists or not.

    Each RemoteResourcesSyncInfo has a main RemoteResource of the same name as the service.
    This is similar to the Service and Resource relationship.
    :param service_id:
    :param session:
    """
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
    main_resource = models.RemoteResource(service_id=service_id,
                                          resource_name=unicode(sync_info.service.resource_name),
                                          resource_type=u"directory")
    session.add(main_resource)
    session.flush()
    sync_info.remote_resource_id = main_resource.resource_id
    session.flush()

def _update_db(remote_resources, service_id, session):
    """
    Writes remote resources to database.
    :param remote_resources:
    :param service_id:
    :param session:
    """
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)

    def add_children(resources, parent_id, position=0):
        for resource_name, values in resources.items():
            new_resource = models.RemoteResource(service_id=sync_info.service_id,
                                                 resource_name=unicode(resource_name),
                                                 resource_type=values['resource_type'],
                                                 parent_id=parent_id,
                                                 ordering=position)
            session.add(new_resource)
            session.flush()
            position += 1
            add_children(values['children'], new_resource.resource_id)

    first_item = list(remote_resources)[0]
    add_children(remote_resources[first_item]['children'], sync_info.remote_resource_id)

    sync_info.last_sync = datetime.datetime.now()

    session.flush()

def _get_resource_children(resource, db_session):
    """
    Mostly copied from ziggurat_foundations to use RemoteResource instead of Resource
    :param resource:
    :param db_session:
    :return:
    """
    query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session)

    def build_subtree_strut(result):
        """
        Returns a dictionary in form of
        {node:Resource, children:{node_id: RemoteResource}}
        """
        items = list(result)
        root_elem = {'node': None, 'children': OrderedDict()}
        if len(items) == 0:
            return root_elem
        for i, node in enumerate(items):
            new_elem = {'node': node.RemoteResource, 'children': OrderedDict()}
            path = list(map(int, node.path.split('/')))
            parent_node = root_elem
            normalized_path = path[:-1]
            if normalized_path:
                for path_part in normalized_path:
                    parent_node = parent_node['children'][path_part]
            parent_node['children'][new_elem['node'].resource_id] = new_elem
        return root_elem

    return build_subtree_strut(query)['children']

def _format_resource_tree(children):
    fmt_res_tree = {}
    for child_id, child_dict in children.items():
        resource = child_dict[u'node']
        new_children = child_dict[u'children']
        resource_dict = {'children': _format_resource_tree(new_children),
                         'resource_type': resource.resource_type}
        fmt_res_tree[resource.resource_name] = resource_dict
    return fmt_res_tree

def _query_remote_resources_in_database(service_type, session):
    """
    Reads remote resources from the RemoteResources table. No external request is made.
    :param service_type:
    :param session:
    :return: a dictionary of the form defined in '_is_valid_resource_schema'
    """
    service = session.query(models.Service).filter_by(type=service_type).first()
    _ensure_sync_info_exists(service.resource_id, session)

    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
    main_resource = session.query(models.RemoteResource).filter_by(
        resource_id=sync_info.remote_resource_id).first()
    tree = _get_resource_children(main_resource, session)

    remote_resources = _format_resource_tree(tree)
    return {service_type: {'children': remote_resources, 'resource_type': 'directory'}}

def get_last_sync(service_type, session):
    last_sync = None
    service = session.query(models.Service).filter_by(type=service_type).first()
    _ensure_sync_info_exists(service.resource_id, session)
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
    if sync_info:
        last_sync = sync_info.last_sync
    return last_sync

def fetch_single_service(service_type, session):
    """
    Get remote resources for a single service.
    :param service_type:
    :param session:
    """
    service = session.query(models.Service).filter_by(type=service_type).first()
    remote_resources = _get_remote_resources(service, service_type)
    service_id = service.resource_id
    _delete_records(service_id, session)
    _ensure_sync_info_exists(service.resource_id, session)
    _update_db(remote_resources, service_id, session)

def fetch():
    """
    Main function to get all remote resources for each service and write to database.
    """
    url = db.get_db_url()
    engine = create_engine(url)

    session = Session(bind=engine)

    for service_name in SYNC_SERVICES:
        fetch_single_service(service_name, session)

    session.commit()
    session.close()

def housekeeping():
    """
    Clean resources that are in the Resource table but have no
    group or user permissions associated to them.
    """
    url = db.get_db_url()
    engine = create_engine(url)

    session = Session(bind=engine)

    for resource in session.query(models.Resource):
        if resource.resource_type_name == 'service':
            continue
        if not resource.group_permissions and not resource.user_permissions:
            session.delete(resource)

    session.commit()
    session.close()

def main():
    """
    Main entry point for cron service.
    """
    fetch()
    housekeeping()

if __name__ == '__main__':
    main()
<commit_msg>implement project-api<commit_after>"""
Sychronize local and remote resources.

To implement a new service, see the _SyncServiceInterface class.
"""

import abc
import copy
import datetime
from collections import OrderedDict

import requests
import threddsclient
from sqlalchemy import create_engine
from sqlalchemy.orm import Session

from magpie import db, models

def merge_local_and_remote_resources(resources_local, service_type, session):
    """Main function to sync resources with remote server"""
    remote_resources = _query_remote_resources_in_database(service_type, session=session)
    merged_resources = _merge_resources(resources_local, remote_resources)
    _sort_resources(merged_resources)
    return merged_resources

def _merge_resources(resources_local, resources_remote):
    """
    Merge resources_local and resources_remote, adding the following keys to the output:

        - remote_path: '/' separated string representing the remote path of the resource
        - matches_remote: True or False depending if the resource is present on the remote server
        - id: set to the value of 'remote_path' if the resource if remote only

    returns a dictionary of the form validated by 'is_valid_resource_schema'

    """
    if not resources_remote:
        return resources_local

    assert _is_valid_resource_schema(resources_local, ignore_resource_type=True)
    assert _is_valid_resource_schema(resources_remote)

    if not resources_local:
        raise ValueError("The resources must contain at least the service name.")

    # The first item is the service name. It is skipped so that only the resources are compared.
    service_name = resources_local.keys()[0]
    _, remote_values = resources_remote.popitem()
    resources_remote = {service_name: remote_values}

    # don't overwrite the input arguments
    merged_resources = copy.deepcopy(resources_local)

    def recurse(_resources_local, _resources_remote, remote_path="", remote_type_path=""):
        # loop local resources, looking for matches in remote resources
        for resource_name_local, values in _resources_local.items():
            current_path = "/".join([remote_path, str(resource_name_local)])

            matches_remote = resource_name_local in _resources_remote
            resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
            current_type_path = "/".join([remote_type_path, resource_type])

            values["remote_path"] = "" if matches_remote else current_path
            values["remote_type_path"] = current_type_path
            values["matches_remote"] = matches_remote
            values["resource_type"] = resource_type

            resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}

            recurse(values['children'], resource_remote_children, current_path, current_type_path)

        # loop remote resources, looking for matches in local resources
        for resource_name_remote, values in _resources_remote.items():
            if resource_name_remote not in _resources_local:
                current_path = "/".join([remote_path, str(resource_name_remote)])
                current_type_path = "/".join([remote_type_path, values['resource_type']])
                new_resource = {'permission_names': [],
                                'children': {},
                                'resource_type': values['resource_type'],
                                'id': current_path,
                                'remote_path': current_path,
                                'remote_type_path': current_type_path,
                                'matches_remote': True}
                _resources_local[resource_name_remote] = new_resource
                recurse(new_resource['children'], values['children'], current_path, current_type_path)

    recurse(merged_resources, resources_remote)

    assert _is_valid_resource_schema(merged_resources)

    return merged_resources

def _is_valid_resource_schema(resources, ignore_resource_type=False):
    """
    Returns True if the structure of the input dictionary is a tree of the form:

    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
                                      'resource_name_4': {'children': {}, 'resource_type': ...}
                                      },
                         'resource_type': ...
                         },
     'resource_name_2': {'children': {}, resource_type': ...}
     }
    :return: bool
    """
    for resource_name, values in resources.items():
        if 'children' not in values:
            return False
        if not ignore_resource_type and 'resource_type' not in values:
            return False
        if not isinstance(values['children'], (OrderedDict, dict)):
            return False
        return _is_valid_resource_schema(values['children'],
                                         ignore_resource_type=ignore_resource_type)
    return True

def _sort_resources(resources):
    """
    Sorts a resource dictionary of the type validated by '_is_valid_resource_schema'
    by using an OrderedDict
    :return: None
    """
    for resource_name, values in resources.items():
        values['children'] = OrderedDict(sorted(values['children'].iteritems()))
        return _sort_resources(values['children'])

class _SyncServiceInterface:
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def get_resources(self):
        """
        This is the function actually fetching the data from the remote service.
        Implement this for every specific service.

        :return: The returned dictionary must be validated by 'is_valid_resource_schema'
        """
        pass

class _SyncServiceGeoserver:
    def __init__(self, geoserver_url):
        self.geoserver_url = geoserver_url

    def get_resources(self):
        # Only workspaces are fetched for now
        resource_type = "route"
        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
        resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
        resp.raise_for_status()
        workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})

        workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}

        resources = {"geoserver-api": {"children": workspaces,
                                       "resource_type": resource_type}}
        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
        return resources

class _SyncServiceProjectAPI:
    def __init__(self, project_api_url):
        self.project_api_url = project_api_url

    def get_resources(self):
        # Only workspaces are fetched for now
        resource_type = "route"
        projects_url = "/".join([self.project_api_url, "api", "Projects"])
        resp = requests.get(projects_url)
        resp.raise_for_status()

        projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}

        resources = {"project-api": {"children": projects, "resource_type": resource_type}}
        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
        return resources

class _SyncServiceThreads(_SyncServiceInterface):
    DEPTH_DEFAULT = 2

    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
        self.thredds_url = thredds_url
        self.depth = depth
        self.kwargs = kwargs  # kwargs is passed to the requests.get method.

    def get_resources(self):
        def thredds_get_resources(url, depth, **kwargs):
            cat = threddsclient.read_url(url, **kwargs)
            name = cat.name
            resource_type = 'directory'
            if cat.datasets and cat.datasets[0].content_type != "application/directory":
                resource_type = 'file'

            tree_item = {name: {'children': {}, 'resource_type': resource_type}}

            if depth > 0:
                for reference in cat.flat_references():
                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))

            return tree_item

        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
        assert _is_valid_resource_schema(resources), 'Error in Interface implementation'
        return resources

class _SyncServiceDefault(_SyncServiceInterface):
    def __init__(self, _):
        pass

    def get_resources(self):
        return {}

SYNC_SERVICES = {
    "thredds": _SyncServiceThreads,
    "geoserver-api": _SyncServiceGeoserver,
    "project-api": _SyncServiceProjectAPI,
}

def _ensure_sync_info_exists(service_resource_id, session):
    """
    Make sure the RemoteResourcesSyncInfo entry exists in the database.
    :param service_resource_id:
    :param session:
    """
    service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_resource_id, session)
    if not service_sync_info:
        sync_info = models.RemoteResourcesSyncInfo(service_id=service_resource_id)
        session.add(sync_info)
        session.flush()
        _create_main_resource(service_resource_id, session)

def _get_remote_resources(service, service_name):
    """
    Request rmeote resources, depending on service type.
    :param service:
    :param service_name:
    :return:
    """
    service_url = service.url
    if service_url.endswith("/"):  # remove trailing slash
        service_url = service_url[:-1]
    sync_service = SYNC_SERVICES.get(service_name.lower(), _SyncServiceDefault)(service_url)
    return sync_service.get_resources()

def _delete_records(service_id, session):
    """
    Delete all RemoteResource based on a Service.resource_id
    :param service_id:
    :param session:
    """
    session.query(models.RemoteResource).filter_by(service_id=service_id).delete()
    session.flush()

def _create_main_resource(service_id, session):
    """
    Creates a main resource for a service, whether one currently exists or not.

    Each RemoteResourcesSyncInfo has a main RemoteResource of the same name as the service.
    This is similar to the Service and Resource relationship.
    :param service_id:
    :param session:
    """
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
    main_resource = models.RemoteResource(service_id=service_id,
                                          resource_name=unicode(sync_info.service.resource_name),
                                          resource_type=u"directory")
    session.add(main_resource)
    session.flush()
    sync_info.remote_resource_id = main_resource.resource_id
    session.flush()

def _update_db(remote_resources, service_id, session):
    """
    Writes remote resources to database.
    :param remote_resources:
    :param service_id:
    :param session:
    """
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)

    def add_children(resources, parent_id, position=0):
        for resource_name, values in resources.items():
            new_resource = models.RemoteResource(service_id=sync_info.service_id,
                                                 resource_name=unicode(resource_name),
                                                 resource_type=values['resource_type'],
                                                 parent_id=parent_id,
                                                 ordering=position)
            session.add(new_resource)
            session.flush()
            position += 1
            add_children(values['children'], new_resource.resource_id)

    first_item = list(remote_resources)[0]
    add_children(remote_resources[first_item]['children'], sync_info.remote_resource_id)

    sync_info.last_sync = datetime.datetime.now()

    session.flush()

def _get_resource_children(resource, db_session):
    """
    Mostly copied from ziggurat_foundations to use RemoteResource instead of Resource
    :param resource:
    :param db_session:
    :return:
    """
    query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session)

    def build_subtree_strut(result):
        """
        Returns a dictionary in form of
        {node:Resource, children:{node_id: RemoteResource}}
        """
        items = list(result)
        root_elem = {'node': None, 'children': OrderedDict()}
        if len(items) == 0:
            return root_elem
        for i, node in enumerate(items):
            new_elem = {'node': node.RemoteResource, 'children': OrderedDict()}
            path = list(map(int, node.path.split('/')))
            parent_node = root_elem
            normalized_path = path[:-1]
            if normalized_path:
                for path_part in normalized_path:
                    parent_node = parent_node['children'][path_part]
            parent_node['children'][new_elem['node'].resource_id] = new_elem
        return root_elem

    return build_subtree_strut(query)['children']

def _format_resource_tree(children):
    fmt_res_tree = {}
    for child_id, child_dict in children.items():
        resource = child_dict[u'node']
        new_children = child_dict[u'children']
        resource_dict = {'children': _format_resource_tree(new_children),
                         'resource_type': resource.resource_type}
        fmt_res_tree[resource.resource_name] = resource_dict
    return fmt_res_tree

def _query_remote_resources_in_database(service_type, session):
    """
    Reads remote resources from the RemoteResources table. No external request is made.
    :param service_type:
    :param session:
    :return: a dictionary of the form defined in '_is_valid_resource_schema'
    """
    service = session.query(models.Service).filter_by(type=service_type).first()
    _ensure_sync_info_exists(service.resource_id, session)

    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
    main_resource = session.query(models.RemoteResource).filter_by(
        resource_id=sync_info.remote_resource_id).first()
    tree = _get_resource_children(main_resource, session)

    remote_resources = _format_resource_tree(tree)
    return {service_type: {'children': remote_resources, 'resource_type': 'directory'}}

def get_last_sync(service_type, session):
    last_sync = None
    service = session.query(models.Service).filter_by(type=service_type).first()
    _ensure_sync_info_exists(service.resource_id, session)
    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
    if sync_info:
        last_sync = sync_info.last_sync
    return last_sync

def fetch_single_service(service_type, session):
    """
    Get remote resources for a single service.
    :param service_type:
    :param session:
    """
    service = session.query(models.Service).filter_by(type=service_type).first()
    remote_resources = _get_remote_resources(service, service_type)
    service_id = service.resource_id
    _delete_records(service_id, session)
    _ensure_sync_info_exists(service.resource_id, session)
    _update_db(remote_resources, service_id, session)

def fetch():
    """
    Main function to get all remote resources for each service and write to database.
    """
    url = db.get_db_url()
    engine = create_engine(url)

    session = Session(bind=engine)

    for service_name in SYNC_SERVICES:
        fetch_single_service(service_name, session)

    session.commit()
    session.close()

def housekeeping():
    """
    Clean resources that are in the Resource table but have no
    group or user permissions associated to them.
    """
    url = db.get_db_url()
    engine = create_engine(url)

    session = Session(bind=engine)

    for resource in session.query(models.Resource):
        if resource.resource_type_name == 'service':
            continue
        if not resource.group_permissions and not resource.user_permissions:
            session.delete(resource)

    session.commit()
    session.close()

def main():
    """
    Main entry point for cron service.
    """
    fetch()
    housekeeping()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import email
import sys
import os
import argparse
import textwrap

def make_person_schema(msg, schemaFile):
  schema = """\

    %s

""" % msg['from']
  schemaFile.write(textwrap.dedent(schema))

def mails2schema(mailDir, outputDir):
  i = 0
  for mail in os.listdir(mailDir):
    mailFilename = mailDir + "/" + mail
    if(os.path.isfile(mailFilename)):
      schemaFilename = "%s/person%d.html" % (outputDir,i)
      i = i + 1
      with open(mailFilename, 'r') as mailFile, open(schemaFilename, 'w') as schemaFile:
        make_person_schema(email.message_from_file(mailFile), schemaFile)

def main():
  parser = argparse.ArgumentParser(description='Mail to schema')
  parser.add_argument('-d', required=True, help='Directory containing mail files (.eml).')
  parser.add_argument('-o', required=True, help='Output directory for the schemas.')
  args = parser.parse_args()
  if not os.path.isdir(args.d):
    print('%s is not a directory (option -d).', args.d)
  elif not os.path.isdir(args.o):
    print('%s is not a directory (option -o).', args.o)
  else:
    mails2schema(args.d, args.o)

if __name__ == "__main__":
  main()
<commit_msg>Correct handling of name and adress encoding.<commit_after>import email
import sys
import os
import argparse
import textwrap
from email.header import decode_header
from email.parser import BytesParser

def make_utf8_str(s):
  res = ""
  for (data, encoding) in decode_header(s):
    if data != None and encoding != None:
      res += str(data.decode(encoding).encode("utf-8"), "utf-8")
    else:
      res += data
  return res

def make_person_schema(mailFile, schemaFile):
  msg = BytesParser().parse(mailFile)
  (realname, mailAddr) = email.utils.parseaddr(msg['from'])
  realname = make_utf8_str(realname)
  mailAddr = make_utf8_str(mailAddr)
  schema = """\

    <spane itemprop="name">%s
    %s

""" % (realname, mailAddr)
  schemaFile.write(textwrap.dedent(schema))

def mails2schema(mailDir, outputDir):
  i = 0
  for mail in os.listdir(mailDir):
    mailFilename = mailDir + "/" + mail
    print(mailFilename)
    if(os.path.isfile(mailFilename)):
      schemaFilename = "%s/person%d.html" % (outputDir,i)
      i = i + 1
      with open(mailFilename, 'r+b') as mailFile, open(schemaFilename, 'w', encoding='utf8') as schemaFile:
        make_person_schema(mailFile, schemaFile)

def main():
  parser = argparse.ArgumentParser(description='Mail to schema')
  parser.add_argument('-d', required=True, help='Directory containing mail files (.eml).')
  parser.add_argument('-o', required=True, help='Output directory for the schemas.')
  args = parser.parse_args()
  if not os.path.isdir(args.d):
    print('%s is not a directory (option -d).', args.d)
  elif not os.path.isdir(args.o):
    print('%s is not a directory (option -o).', args.o)
  else:
    mails2schema(args.d, args.o)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>from clint.packages import colorama

"""
Simple colorama wrapper
"""

colorama.init(autoreset=True)

class ColoredString(object):
    def __init__(self, color, s):
        super(ColoredString, self).__init__()
        self.s = s
        self.color = color
        self.color_str = '%s%s%s' % (
            getattr(colorama.Fore, self.color), self.s, colorama.Fore.RESET)

    def __len__(self):
        return len(self.s)

    def __repr__(self):
        return "<%s-string: '%s'>" % (self.color, self.s)

    def __str__(self):
        return self.__unicode__().encode('utf8')

    def __unicode__(self):
        return self.color_str

    def __add__(self, other):
        return str(self.color_str) + str(other)

    def __radd__(self, other):
        return str(other) + str(self.color_str)

    def __mul__(self, other):
        return (self.color_str * other)

def black(string):
    return ColoredString('BLACK', string)

def red(string):
	return ColoredString('RED', string)

def green(string):
	return ColoredString('GREEN', string)

def yellow(string):
	return ColoredString('YELLOW', string)

def blue(string):
	return ColoredString('BLUE', string)

def magenta(string):
	return ColoredString('MAGENTA', string)

def cyan(string):
	return ColoredString('CYAN', string)

def white(string):
	return ColoredString('STRING', string)
<commit_msg>No one needs that class.<commit_after>from clint.packages import colorama

"""
Simple colorama wrapper
"""

colorama.init(autoreset=True)

__all__ = (
    'black', 'red', 'green', 'yellow', 'blue',
    'black', 'megenta', 'cyan', 'white'
)

class ColoredString(object):
    def __init__(self, color, s):
        super(ColoredString, self).__init__()
        self.s = s
        self.color = color
        self.color_str = '%s%s%s' % (
            getattr(colorama.Fore, self.color), self.s, colorama.Fore.RESET)

    def __len__(self):
        return len(self.s)

    def __repr__(self):
        return "<%s-string: '%s'>" % (self.color, self.s)

    def __str__(self):
        return self.__unicode__().encode('utf8')

    def __unicode__(self):
        return self.color_str

    def __add__(self, other):
        return str(self.color_str) + str(other)

    def __radd__(self, other):
        return str(other) + str(self.color_str)

    def __mul__(self, other):
        return (self.color_str * other)

def black(string):
    return ColoredString('BLACK', string)

def red(string):
	return ColoredString('RED', string)

def green(string):
	return ColoredString('GREEN', string)

def yellow(string):
	return ColoredString('YELLOW', string)

def blue(string):
	return ColoredString('BLUE', string)

def magenta(string):
	return ColoredString('MAGENTA', string)

def cyan(string):
	return ColoredString('CYAN', string)

def white(string):
	return ColoredString('STRING', string)
<|endoftext|>


<commit_before>from z3 import *
from mythril.analysis.ops import *
from mythril.analysis.report import Issue
import re
import logging

'''
MODULE DESCRIPTION:

Check for SUICIDE instructions that are not constrained by caller.
'''

def execute(statespace):

    issues = []

    svm = statespace.svm

    for k in svm.nodes:
        node = svm.nodes[k]

        for instruction in node.instruction_list:

            if(instruction['opcode'] == "SUICIDE"):

                issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. "

                state = node.states[instruction['address']]
                to = state.stack.pop()

                constraint_on_caller = None

                for constraint in node.constraints:
                    if "caller" in str(constraint):
                        m = re.search(r'storage_([0-9a-f])+', str(constraint))
                        cc_storage_index = m.group(1)
                        constraint_on_caller = constraint
                        break

                can_solve = False

                if constraint_on_caller is not None:

                    logging.info("caller constraint storage index " + cc_storage_index)

                    # Is there any way to write to that storage slot?

                    try:
                        sstors = statespace.tainted_sstors[cc_storage_index]

                        for sstor in sstors:

                            s = Solver()

                            for constraint in sstor.node.constraints:
                                s.add(constraint)

                            if (s.check() == sat):
                                m = s.model()
                                can_solve = True
                                break
                    except KeyError:
                        logging.info("No viable storage writes to index " + cc_storage_index)
                        can_solve = False

                else:
                    can_solve = True

                if can_solve:

                    for constraint in node.constraints:
                        s.add(constraint)

                    if (s.check() == sat):
                        issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                        issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. It appears as if the function does not check the caller address.\n"

                        if ("caller" in str(to)):
                            issue.description += "The remaining Ether is sent to the caller's address.\n"
                        elif ("storage" in str(to)):
                            issue.description += "The remaining Ether is sent to a stored address\n"
                        else:
                            issue.description += "The remaining Ether is sent to: " + str(to) + "\n"

                        issue.description += "Solver output:\n"

                        m = s.model()

                        for d in m.decls():
                            issue.description += "%s = 0x%x\n" % (d.name(), m[d].as_long())

                    issues.append(issue)

    return issues

<commit_msg>Refactor<commit_after>from z3 import *
from mythril.analysis.ops import *
from mythril.analysis.report import Issue
import re
import logging

'''
MODULE DESCRIPTION:

Check for SUICIDE instructions that are not constrained by caller
'''

def execute(statespace):

    issues = []

    svm = statespace.svm

    for k in svm.nodes:
        node = svm.nodes[k]

        for instruction in node.instruction_list:

            if(instruction['opcode'] == "SUICIDE"):

                issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. "

                state = node.states[instruction['address']]
                to = state.stack.pop()

                constraint_on_caller = None

                for constraint in node.constraints:
                    if "caller" in str(constraint):
                        m = re.search(r'storage_([0-9a-f])+', str(constraint))
                        cc_storage_index = m.group(1)
                        constraint_on_caller = constraint
                        break

                can_solve = False

                if constraint_on_caller is not None:

                    logging.info("caller constraint storage index " + cc_storage_index)

                    # Is there any way to write to that storage slot?

                    try:
                        sstors = statespace.tainted_sstors[cc_storage_index]

                        for sstor in sstors:

                            s = Solver()

                            for constraint in sstor.node.constraints:
                                s.add(constraint)

                            if (s.check() == sat):
                                m = s.model()
                                can_solve = True
                                break
                    except KeyError:
                        logging.info("No viable storage writes to index " + cc_storage_index)
                        can_solve = False

                else:
                    can_solve = True

                if can_solve:

                    for constraint in node.constraints:
                        s.add(constraint)

                    if (s.check() == sat):
                        issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                        issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. It appears as if the function does not check the caller address.\n"

                        if ("caller" in str(to)):
                            issue.description += "The remaining Ether is sent to the caller's address.\n"
                        elif ("storage" in str(to)):
                            issue.description += "The remaining Ether is sent to a stored address\n"
                        else:
                            issue.description += "The remaining Ether is sent to: " + str(to) + "\n"

                        issue.description += "Solver output:\n"

                        m = s.model()

                        for d in m.decls():
                            issue.description += "%s = 0x%x\n" % (d.name(), m[d].as_long())

                    issues.append(issue)

    return issues
<|endoftext|>


<commit_before>from z3 import *
from mythril.analysis.ops import *
from mythril.analysis.report import Issue
import re
import logging

'''
MODULE DESCRIPTION:

Check for SUICIDE instructions that are not constrained by caller
'''

def execute(statespace):

    issues = []

    svm = statespace.svm

    for k in svm.nodes:
        node = svm.nodes[k]

        for instruction in node.instruction_list:

            if(instruction['opcode'] == "SUICIDE"):

                issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. "

                state = node.states[instruction['address']]
                to = state.stack.pop()

                constraint_on_caller = None

                for constraint in node.constraints:
                    if "caller" in str(constraint):
                        m = re.search(r'storage_([0-9a-f])+', str(constraint))
                        cc_storage_index = m.group(1)
                        constraint_on_caller = constraint
                        break

                can_solve = False

                if constraint_on_caller is not None:

                    logging.info("caller constraint storage index " + cc_storage_index)

                    # Is there any way to write to that storage slot?

                    try:
                        sstors = statespace.tainted_sstors[cc_storage_index]

                        for sstor in sstors:

                            s = Solver()

                            for constraint in sstor.node.constraints:
                                s.add(constraint)

                            if (s.check() == sat):
                                m = s.model()
                                can_solve = True
                                break
                    except KeyError:
                        logging.info("No viable storage writes to index " + cc_storage_index)
                        can_solve = False

                else:
                    can_solve = True

                if can_solve:

                    for constraint in node.constraints:
                        s.add(constraint)

                    if (s.check() == sat):
                        issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                        issue.description = "The function " + node.function_name + " calls the SUICIDE instruction. It appears as if the function does not check the caller address.\n"

                        if ("caller" in str(to)):
                            issue.description += "The remaining Ether is sent to the caller's address.\n"
                        elif ("storage" in str(to)):
                            issue.description += "The remaining Ether is sent to a stored address\n"
                        else:
                            issue.description += "The remaining Ether is sent to: " + str(to) + "\n"

                        issue.description += "Solver output:\n"

                        m = s.model()

                        for d in m.decls():
                            issue.description += "%s = 0x%x\n" % (d.name(), m[d].as_long())

                    issues.append(issue)

    return issues
<commit_msg>Update SUICIDE module<commit_after>from z3 import *
from mythril.analysis.ops import *
from mythril.analysis.report import Issue
import re
import logging

'''
MODULE DESCRIPTION:

Check for SUICIDE instructions that can be triggered by anyone
'''

def execute(statespace):

    issues = []

    svm = statespace.svm

    for k in svm.nodes:
        node = svm.nodes[k]

        for instruction in node.instruction_list:

            if(instruction['opcode'] == "SUICIDE"):

                issue = Issue("Unchecked SUICIDE", "VULNERABILITY")
                issue.description = "The function " + node.function_name + " executes the SUICIDE instruction."

                state = node.states[instruction['address']]
                to = state.stack.pop()

                constraint_on_caller = None

                for constraint in node.constraints:
                    if "caller" in str(constraint):
                        m = re.search(r'storage_([0-9a-f])+', str(constraint))
                        cc_storage_index = m.group(1)
                        constraint_on_caller = constraint
                        break

                can_solve = False

                if constraint_on_caller is not None:

                    logging.info("caller constraint storage index " + cc_storage_index)

                    # Is there any way to write to that storage slot?

                    try:
                        sstors = statespace.tainted_sstors[cc_storage_index]

                        for sstor in sstors:

                            s = Solver()

                            for constraint in sstor.node.constraints:
                                s.add(constraint)

                            if (s.check() == sat):
                                m = s.model()

                                issue.description += "The transaction sender is checked against a value in storage index " + str(cc_storage_index) + ". However, it may be possible to overwrite this value by calling the function '" + sstor.node.function_name + "'."

                                can_solve = True
                                break
                    except KeyError:
                        logging.debug("No viable storage writes to index " + cc_storage_index)
                        can_solve = False

                else:
                    issue.description = "There are no restrictions on msg.sender."
                    can_solve = True

                if can_solve:

                    s = Solver()

                    for constraint in node.constraints:
                        s.add(constraint)

                    if (s.check() == sat):

                        if ("caller" in str(to)):
                            issue.description += "\nThe remaining Ether is sent to the caller's address.\n"
                        elif ("storage" in str(to)):
                            issue.description += "\nThe remaining Ether is sent to a stored address\n"
                        else:
                            issue.description += "\nThe remaining Ether is sent to: " + str(to) + "\n"

                        issue.description += "Solver output:\n"

                        m = s.model()

                        logging.debug("Model for node " + str(k) + ", function " +  svm.nodes[k].function_name + ": ")

                        for d in m.decls():
                            logging.debug("%s = 0x%x" % (d.name(), m[d].as_long()))

                        issues.append(issue)

    return issues
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2019-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import json
import os
import re

from six import iteritems

from datadog_checks.base import AgentCheck, ensure_unicode
from datadog_checks.base.errors import CheckException

from .common import ALLOWED_METRICS, COUNT_METRICS, GAUGE_METRICS, HISTOGRAM_METRICS, MONOTONIC_COUNTER_METRICS
from .utils import get_fqdn, get_stream_id_for_topic

try:
    # This is not the regular `confluent_kafka` but rather the custom version made by MapR
    # that you can install with "pip install mapr-streams-python"
    import confluent_kafka as ck

    ck_import_error = None
except ImportError as e:
    ck = None
    ck_import_error = e

DEFAULT_STREAM_PATH = "/var/mapr/mapr.monitoring/metricstreams"
METRICS_SUBMITTED_METRIC_NAME = "mapr.metrics.submitted"
SERVICE_CHECK = "mapr.can_connect"
TICKET_LOCATION_ENV_VAR = 'MAPR_TICKETFILE_LOCATION'

"""
All mapr metrics go through a Stream topic (similar to a Kafka topic) and are consumed by OpenTSDB, a timeseries
database.
All the metrics are distributed over multiple topics, one for each host. This way, the check instance can subscribe
to the topic relevant to the current host and consume everything.

Note: The MapR documentation https://mapr.com/docs/61/AdministratorGuide/spyglass-on-streams.html states that
there is one topic per host per metric name. This is no longer true starting with 6.1+, there is only one topic per
host now. To support older versions of MapR, the check should be updated to subscribe those multiple topics.
"""

class MaprCheck(AgentCheck):
    def __init__(self, name, init_config, instances):
        super(MaprCheck, self).__init__(name, init_config, instances)
        self._conn = None
        self.hostname = self.instance.get('hostname', get_fqdn())
        self.streams_count = self.instance.get('streams_count', 1)
        self.topic_path = "{stream_path}/{stream_id}:{topic_name}".format(
            stream_path=self.instance.get('stream_path', DEFAULT_STREAM_PATH),
            stream_id=get_stream_id_for_topic(self.hostname, rng=self.streams_count),
            topic_name=self.hostname,
        )
        self.allowed_metrics = [re.compile(w) for w in self.instance.get('metric_whitelist', [])]
        self.base_tags = self.instance.get('tags', [])
        self.has_ever_submitted_metrics = False

        auth_ticket = self.instance.get('ticket_location', os.environ.get(TICKET_LOCATION_ENV_VAR))

        if not auth_ticket:
            self.log.warning(
                "Neither `ticket_location` (in the config.yaml) or the %s environment variable is set. This will"
                "cause authentication issues if your cluster requires authenticated requests.",
                TICKET_LOCATION_ENV_VAR,
            )
        elif not os.access(auth_ticket, os.R_OK):
            raise CheckException(
                "MapR authentication ticket located at %s is not readable by the dd-agent "
                "user. Please update the file permissions.",
                auth_ticket,
            )

        os.environ[TICKET_LOCATION_ENV_VAR] = auth_ticket

    def check(self, _):
        if ck is None:
            raise CheckException(
                "confluent_kafka was not imported correctly, make sure the library is installed and that you've "
                "set LD_LIBRARY_PATH correctly. Please refer to datadog documentation for more details. Error is %s"
                % ck_import_error
            )

        try:
            conn = self.get_connection()
        except Exception:
            self.service_check(
                SERVICE_CHECK, AgentCheck.CRITICAL, self.base_tags + ['topic:{}'.format(self.topic_path)]
            )
            raise
        else:
            self.service_check(SERVICE_CHECK, AgentCheck.OK, self.base_tags + ['topic:{}'.format(self.topic_path)])

        submitted_metrics_count = 0

        while True:
            # Collecting one message at a time has no impact on performance because the library
            # batches data. Most calls to `poll` won't initiate a I/O connection.
            msg = conn.poll(timeout=0.5)
            if msg is None:
                # Timed out, no more messages
                break

            if msg.error() is None:
                # Metric received
                try:
                    metric = json.loads(ensure_unicode(msg.value()))[0]
                    metric_name = metric['metric']
                    if self.should_collect_metric(metric_name):
                        # Will sometimes submit the same metric multiple time, but because it's only
                        # gauges and monotonic_counter that's fine.
                        self.submit_metric(metric)
                        submitted_metrics_count += 1
                except Exception as e:
                    self.log.warning("Received unexpected message %s, wont be processed", msg.value())
                    self.log.exception(e)
            elif msg.error().code() == ck.KafkaError.TOPIC_AUTHORIZATION_FAILED:
                raise CheckException(
                    "The user impersonated using the ticket %s does not have the 'consume' permission on topic %s. "
                    "Please update the stream permissions." % (self.auth_ticket, self.topic_path)
                )
            elif msg.error().code() != ck.KafkaError._PARTITION_EOF:
                # Partition EOF is expected anytime we reach the end of one partition in the topic.
                # This is expected at least once per partition per check run.
                raise CheckException(msg.error())

        if not self.has_ever_submitted_metrics:
            # The integration has never found any metric so far
            if submitted_metrics_count:
                self.has_ever_submitted_metrics = True
                self.log.info("The integration collected metrics for the first time in topic %s", self.topic_path)
            else:
                self.log.error(
                    "The integration was not yet able to collect any MapR metric in topic %s. If this error continues "
                    "after a few check runs, double-check the existence of the stream and the topic using "
                    "maprcli as well as the permissions on this topic.",
                    self.topic_path,
                )

        if submitted_metrics_count:
            self.gauge(METRICS_SUBMITTED_METRIC_NAME, submitted_metrics_count, self.base_tags)

    def get_connection(self):
        if self._conn:
            # confluent_kafka takes care of recreating the connection if anything goes wrong.
            return self._conn

        self._conn = ck.Consumer(
            {
                "group.id": "dd-agent",  # uniquely identify this consumer
                "enable.auto.commit": False  # important, do not store the offset for this consumer,
                # if we do it just once the mapr library has a bug (feature?) which prevents resetting it to the head
            }
        )
        self._conn.subscribe([self.topic_path])
        return self._conn

    def should_collect_metric(self, metric_name):
        if metric_name not in ALLOWED_METRICS:
            # Metric is not part of datadog allowed list
            return False
        if not self.allowed_metrics:
            # No filter specified, allow everything
            return True

        for reg in self.allowed_metrics:
            if re.match(reg, metric_name):
                # Metric matched one pattern
                return True

        self.log.debug("Ignoring non whitelisted metric: %s", metric_name)
        return False

    def submit_metric(self, metric):
        metric_name = metric['metric']
        tags = self.base_tags + ["{}:{}".format(k, v) for k, v in iteritems(metric['tags'])]

        if 'buckets' in metric and metric_name in HISTOGRAM_METRICS:
            for bounds, value in metric['buckets'].items():
                lower, upper = bounds.split(',')
                self.submit_histogram_bucket(
                    metric_name, value, int(lower), int(upper), monotonic=False, hostname=self.hostname, tags=tags
                )
        else:
            if metric_name in GAUGE_METRICS:
                self.gauge(metric_name, metric['value'], tags=tags)
            elif metric_name in MONOTONIC_COUNTER_METRICS:
                self.monotonic_count(metric_name, metric['value'], tags=tags)
            elif metric_name in COUNT_METRICS:
                self.count(metric_name, metric['value'], tags=tags)
<commit_msg>Fixup for an AttributeError (#9343)<commit_after># (C) Datadog, Inc. 2019-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import json
import os
import re

from six import iteritems

from datadog_checks.base import AgentCheck, ensure_unicode
from datadog_checks.base.errors import CheckException

from .common import ALLOWED_METRICS, COUNT_METRICS, GAUGE_METRICS, HISTOGRAM_METRICS, MONOTONIC_COUNTER_METRICS
from .utils import get_fqdn, get_stream_id_for_topic

try:
    # This is not the regular `confluent_kafka` but rather the custom version made by MapR
    # that you can install with "pip install mapr-streams-python"
    import confluent_kafka as ck

    ck_import_error = None
except ImportError as e:
    ck = None
    ck_import_error = e

DEFAULT_STREAM_PATH = "/var/mapr/mapr.monitoring/metricstreams"
METRICS_SUBMITTED_METRIC_NAME = "mapr.metrics.submitted"
SERVICE_CHECK = "mapr.can_connect"
TICKET_LOCATION_ENV_VAR = 'MAPR_TICKETFILE_LOCATION'

"""
All mapr metrics go through a Stream topic (similar to a Kafka topic) and are consumed by OpenTSDB, a timeseries
database.
All the metrics are distributed over multiple topics, one for each host. This way, the check instance can subscribe
to the topic relevant to the current host and consume everything.

Note: The MapR documentation https://mapr.com/docs/61/AdministratorGuide/spyglass-on-streams.html states that
there is one topic per host per metric name. This is no longer true starting with 6.1+, there is only one topic per
host now. To support older versions of MapR, the check should be updated to subscribe those multiple topics.
"""

class MaprCheck(AgentCheck):
    def __init__(self, name, init_config, instances):
        super(MaprCheck, self).__init__(name, init_config, instances)
        self._conn = None
        self.hostname = self.instance.get('hostname', get_fqdn())
        self.streams_count = self.instance.get('streams_count', 1)
        self.topic_path = "{stream_path}/{stream_id}:{topic_name}".format(
            stream_path=self.instance.get('stream_path', DEFAULT_STREAM_PATH),
            stream_id=get_stream_id_for_topic(self.hostname, rng=self.streams_count),
            topic_name=self.hostname,
        )
        self.allowed_metrics = [re.compile(w) for w in self.instance.get('metric_whitelist', [])]
        self.base_tags = self.instance.get('tags', [])
        self.has_ever_submitted_metrics = False

        self.auth_ticket = self.instance.get('ticket_location', os.environ.get(TICKET_LOCATION_ENV_VAR))

        if not self.auth_ticket:
            self.log.warning(
                "Neither `ticket_location` (in the config.yaml) or the %s environment variable is set. This will"
                "cause authentication issues if your cluster requires authenticated requests.",
                TICKET_LOCATION_ENV_VAR,
            )
        elif not os.access(self.auth_ticket, os.R_OK):
            raise CheckException(
                "MapR authentication ticket located at %s is not readable by the dd-agent "
                "user. Please update the file permissions.",
                self.auth_ticket,
            )

        os.environ[TICKET_LOCATION_ENV_VAR] = self.auth_ticket

    def check(self, _):
        if ck is None:
            raise CheckException(
                "confluent_kafka was not imported correctly, make sure the library is installed and that you've "
                "set LD_LIBRARY_PATH correctly. Please refer to datadog documentation for more details. Error is %s"
                % ck_import_error
            )

        try:
            conn = self.get_connection()
        except Exception:
            self.service_check(
                SERVICE_CHECK, AgentCheck.CRITICAL, self.base_tags + ['topic:{}'.format(self.topic_path)]
            )
            raise
        else:
            self.service_check(SERVICE_CHECK, AgentCheck.OK, self.base_tags + ['topic:{}'.format(self.topic_path)])

        submitted_metrics_count = 0

        while True:
            # Collecting one message at a time has no impact on performance because the library
            # batches data. Most calls to `poll` won't initiate a I/O connection.
            msg = conn.poll(timeout=0.5)
            if msg is None:
                # Timed out, no more messages
                break

            if msg.error() is None:
                # Metric received
                try:
                    metric = json.loads(ensure_unicode(msg.value()))[0]
                    metric_name = metric['metric']
                    if self.should_collect_metric(metric_name):
                        # Will sometimes submit the same metric multiple time, but because it's only
                        # gauges and monotonic_counter that's fine.
                        self.submit_metric(metric)
                        submitted_metrics_count += 1
                except Exception as e:
                    self.log.warning("Received unexpected message %s, wont be processed", msg.value())
                    self.log.exception(e)
            elif msg.error().code() == ck.KafkaError.TOPIC_AUTHORIZATION_FAILED:
                raise CheckException(
                    "The user impersonated using the ticket %s does not have the 'consume' permission on topic %s. "
                    "Please update the stream permissions." % (self.auth_ticket, self.topic_path)
                )
            elif msg.error().code() != ck.KafkaError._PARTITION_EOF:
                # Partition EOF is expected anytime we reach the end of one partition in the topic.
                # This is expected at least once per partition per check run.
                raise CheckException(msg.error())

        if not self.has_ever_submitted_metrics:
            # The integration has never found any metric so far
            if submitted_metrics_count:
                self.has_ever_submitted_metrics = True
                self.log.info("The integration collected metrics for the first time in topic %s", self.topic_path)
            else:
                self.log.error(
                    "The integration was not yet able to collect any MapR metric in topic %s. If this error continues "
                    "after a few check runs, double-check the existence of the stream and the topic using "
                    "maprcli as well as the permissions on this topic.",
                    self.topic_path,
                )

        if submitted_metrics_count:
            self.gauge(METRICS_SUBMITTED_METRIC_NAME, submitted_metrics_count, self.base_tags)

    def get_connection(self):
        if self._conn:
            # confluent_kafka takes care of recreating the connection if anything goes wrong.
            return self._conn

        self._conn = ck.Consumer(
            {
                "group.id": "dd-agent",  # uniquely identify this consumer
                "enable.auto.commit": False  # important, do not store the offset for this consumer,
                # if we do it just once the mapr library has a bug (feature?) which prevents resetting it to the head
            }
        )
        self._conn.subscribe([self.topic_path])
        return self._conn

    def should_collect_metric(self, metric_name):
        if metric_name not in ALLOWED_METRICS:
            # Metric is not part of datadog allowed list
            return False
        if not self.allowed_metrics:
            # No filter specified, allow everything
            return True

        for reg in self.allowed_metrics:
            if re.match(reg, metric_name):
                # Metric matched one pattern
                return True

        self.log.debug("Ignoring non whitelisted metric: %s", metric_name)
        return False

    def submit_metric(self, metric):
        metric_name = metric['metric']
        tags = self.base_tags + ["{}:{}".format(k, v) for k, v in iteritems(metric['tags'])]

        if 'buckets' in metric and metric_name in HISTOGRAM_METRICS:
            for bounds, value in metric['buckets'].items():
                lower, upper = bounds.split(',')
                self.submit_histogram_bucket(
                    metric_name, value, int(lower), int(upper), monotonic=False, hostname=self.hostname, tags=tags
                )
        else:
            if metric_name in GAUGE_METRICS:
                self.gauge(metric_name, metric['value'], tags=tags)
            elif metric_name in MONOTONIC_COUNTER_METRICS:
                self.monotonic_count(metric_name, metric['value'], tags=tags)
            elif metric_name in COUNT_METRICS:
                self.count(metric_name, metric['value'], tags=tags)
<|endoftext|>