<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import admin

from import_export.admin import ImportExportModelAdmin
from import_export.resources import ModelResource
from import_export.fields import Field
import import_export.widgets as widgets

from .models import PartCategory, Part
from .models import PartAttachment, PartStar
from .models import BomItem
from .models import PartParameterTemplate, PartParameter

from stock.models import StockLocation
from company.models import SupplierPart

class PartResource(ModelResource):
    """ Class for managing Part data import/export """

    # ForeignKey fields
    category = Field(attribute='category', widget=widgets.ForeignKeyWidget(PartCategory))

    default_location = Field(attribute='default_location', widget=widgets.ForeignKeyWidget(StockLocation))

    default_supplier = Field(attribute='default_supplier', widget=widgets.ForeignKeyWidget(SupplierPart))

    category_name = Field(attribute='category__name', readonly=True)

    variant_of = Field(attribute='variant_of', widget=widgets.ForeignKeyWidget(Part))

    suppliers = Field(attribute='supplier_count', readonly=True)

    # Extra calculated meta-data (readonly)
    in_stock = Field(attribute='total_stock', readonly=True, widget=widgets.IntegerWidget())

    on_order = Field(attribute='on_order', readonly=True, widget=widgets.IntegerWidget())

    used_in = Field(attribute='used_in_count', readonly=True, widget=widgets.IntegerWidget())

    allocated = Field(attribute='allocation_count', readonly=True, widget=widgets.IntegerWidget())

    building = Field(attribute='quantity_being_built', readonly=True, widget=widgets.IntegerWidget())

    class Meta:
        model = Part
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True
        exclude = [
            'bom_checksum', 'bom_checked_by', 'bom_checked_date'
        ]

    def get_queryset(self):
        """ Prefetch related data for quicker access """

        query = super().get_queryset()
        query = query.prefetch_related(
            'category',
            'used_in',
            'builds',
            'supplier_parts__purchase_order_line_items',
            'stock_items__allocations'
        )

        return query

class PartAdmin(ImportExportModelAdmin):

    resource_class = PartResource

    list_display = ('full_name', 'description', 'total_stock', 'category')

    list_filter = ('active', 'assembly', 'is_template', 'virtual')

    search_fields = ('name', 'description', 'category__name', 'category__description', 'IPN')

class PartCategoryResource(ModelResource):
    """ Class for managing PartCategory data import/export """

    parent = Field(attribute='parent', widget=widgets.ForeignKeyWidget(PartCategory))

    parent_name = Field(attribute='parent__name', readonly=True)

    default_location = Field(attribute='default_location', widget=widgets.ForeignKeyWidget(StockLocation))

    class Meta:
        model = PartCategory
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True

        exclude = [
            # Exclude MPTT internal model fields
            'lft', 'rght', 'tree_id', 'level',
        ]

    def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):

        super().after_import(dataset, result, using_transactions, dry_run, **kwargs)

        # Rebuild the PartCategory tree(s)
        PartCategory.objects.rebuild()

class PartCategoryAdmin(ImportExportModelAdmin):

    resource_class = PartCategoryResource

    list_display = ('name', 'pathstring', 'description')

    search_fields = ('name', 'description')

class PartAttachmentAdmin(admin.ModelAdmin):

    list_display = ('part', 'attachment', 'comment')

class PartStarAdmin(admin.ModelAdmin):

    list_display = ('part', 'user')

class BomItemResource(ModelResource):
    """ Class for managing BomItem data import/export """

    part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

    part_name = Field(attribute='part__name', readonly=True)

    sub_part = Field(attribute='sub_part', widget=widgets.ForeignKeyWidget(Part))

    sub_part_name = Field(attribute='sub_part__name', readonly=True)

    stock = Field(attribute='sub_part__total_stock', readonly=True)

    class Meta:
        model = BomItem
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True

        exclude = ('checksum')

class BomItemAdmin(ImportExportModelAdmin):

    resource_class = BomItemResource

    list_display = ('part', 'sub_part', 'quantity')

    search_fields = ('part__name', 'part__description', 'sub_part__name', 'sub_part__description')

class ParameterTemplateAdmin(ImportExportModelAdmin):
    list_display = ('name', 'units')

class ParameterResource(ModelResource):
    """ Class for managing PartParameter data import/export """

    part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

    part_name = Field(attribute='part__name', readonly=True)

    template = Field(attribute='template', widget=widgets.ForeignKeyWidget(PartParameterTemplate))

    template_name = Field(attribute='template__name', readonly=True)

    class Meta:
        model = PartParameter
        skip_unchanged = True
        report_skipped = False
        clean_model_instance = True

class ParameterAdmin(ImportExportModelAdmin):

    resource_class = ParameterResource

    list_display = ('part', 'template', 'data')

admin.site.register(Part, PartAdmin)
admin.site.register(PartCategory, PartCategoryAdmin)
admin.site.register(PartAttachment, PartAttachmentAdmin)
admin.site.register(PartStar, PartStarAdmin)
admin.site.register(BomItem, BomItemAdmin)
admin.site.register(PartParameterTemplate, ParameterTemplateAdmin)
admin.site.register(PartParameter, ParameterAdmin)
<commit_msg>Export full_name for parts<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import admin

from import_export.admin import ImportExportModelAdmin
from import_export.resources import ModelResource
from import_export.fields import Field
import import_export.widgets as widgets

from .models import PartCategory, Part
from .models import PartAttachment, PartStar
from .models import BomItem
from .models import PartParameterTemplate, PartParameter

from stock.models import StockLocation
from company.models import SupplierPart

class PartResource(ModelResource):
    """ Class for managing Part data import/export """

    # ForeignKey fields
    category = Field(attribute='category', widget=widgets.ForeignKeyWidget(PartCategory))

    default_location = Field(attribute='default_location', widget=widgets.ForeignKeyWidget(StockLocation))

    default_supplier = Field(attribute='default_supplier', widget=widgets.ForeignKeyWidget(SupplierPart))

    category_name = Field(attribute='category__name', readonly=True)

    variant_of = Field(attribute='variant_of', widget=widgets.ForeignKeyWidget(Part))

    suppliers = Field(attribute='supplier_count', readonly=True)

    # Extra calculated meta-data (readonly)
    in_stock = Field(attribute='total_stock', readonly=True, widget=widgets.IntegerWidget())

    on_order = Field(attribute='on_order', readonly=True, widget=widgets.IntegerWidget())

    used_in = Field(attribute='used_in_count', readonly=True, widget=widgets.IntegerWidget())

    allocated = Field(attribute='allocation_count', readonly=True, widget=widgets.IntegerWidget())

    building = Field(attribute='quantity_being_built', readonly=True, widget=widgets.IntegerWidget())

    class Meta:
        model = Part
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True
        exclude = [
            'bom_checksum', 'bom_checked_by', 'bom_checked_date'
        ]

    def get_queryset(self):
        """ Prefetch related data for quicker access """

        query = super().get_queryset()
        query = query.prefetch_related(
            'category',
            'used_in',
            'builds',
            'supplier_parts__purchase_order_line_items',
            'stock_items__allocations'
        )

        return query

class PartAdmin(ImportExportModelAdmin):

    resource_class = PartResource

    list_display = ('full_name', 'description', 'total_stock', 'category')

    list_filter = ('active', 'assembly', 'is_template', 'virtual')

    search_fields = ('name', 'description', 'category__name', 'category__description', 'IPN')

class PartCategoryResource(ModelResource):
    """ Class for managing PartCategory data import/export """

    parent = Field(attribute='parent', widget=widgets.ForeignKeyWidget(PartCategory))

    parent_name = Field(attribute='parent__name', readonly=True)

    default_location = Field(attribute='default_location', widget=widgets.ForeignKeyWidget(StockLocation))

    class Meta:
        model = PartCategory
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True

        exclude = [
            # Exclude MPTT internal model fields
            'lft', 'rght', 'tree_id', 'level',
        ]

    def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):

        super().after_import(dataset, result, using_transactions, dry_run, **kwargs)

        # Rebuild the PartCategory tree(s)
        PartCategory.objects.rebuild()

class PartCategoryAdmin(ImportExportModelAdmin):

    resource_class = PartCategoryResource

    list_display = ('name', 'pathstring', 'description')

    search_fields = ('name', 'description')

class PartAttachmentAdmin(admin.ModelAdmin):

    list_display = ('part', 'attachment', 'comment')

class PartStarAdmin(admin.ModelAdmin):

    list_display = ('part', 'user')

class BomItemResource(ModelResource):
    """ Class for managing BomItem data import/export """

    part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

    part_name = Field(attribute='part__full_name', readonly=True)

    sub_part = Field(attribute='sub_part', widget=widgets.ForeignKeyWidget(Part))

    sub_part_name = Field(attribute='sub_part__full_name', readonly=True)

    stock = Field(attribute='sub_part__total_stock', readonly=True)

    class Meta:
        model = BomItem
        skip_unchanged = True
        report_skipped = False
        clean_model_instances = True

        exclude = ('checksum')

class BomItemAdmin(ImportExportModelAdmin):

    resource_class = BomItemResource

    list_display = ('part', 'sub_part', 'quantity')

    search_fields = ('part__name', 'part__description', 'sub_part__name', 'sub_part__description')

class ParameterTemplateAdmin(ImportExportModelAdmin):
    list_display = ('name', 'units')

class ParameterResource(ModelResource):
    """ Class for managing PartParameter data import/export """

    part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

    part_name = Field(attribute='part__name', readonly=True)

    template = Field(attribute='template', widget=widgets.ForeignKeyWidget(PartParameterTemplate))

    template_name = Field(attribute='template__name', readonly=True)

    class Meta:
        model = PartParameter
        skip_unchanged = True
        report_skipped = False
        clean_model_instance = True

class ParameterAdmin(ImportExportModelAdmin):

    resource_class = ParameterResource

    list_display = ('part', 'template', 'data')

admin.site.register(Part, PartAdmin)
admin.site.register(PartCategory, PartCategoryAdmin)
admin.site.register(PartAttachment, PartAttachmentAdmin)
admin.site.register(PartStar, PartStarAdmin)
admin.site.register(BomItem, BomItemAdmin)
admin.site.register(PartParameterTemplate, ParameterTemplateAdmin)
admin.site.register(PartParameter, ParameterAdmin)
<|endoftext|>


<commit_before>import os
import re
import sys
import json
import time
import click
import importlib
import traceback
import threading
import subprocess
import watchdog
import anchore_engine.configuration.localconfig
#import anchore_engine.db.entities.upgrade

from watchdog.observers import Observer
from watchdog.events import RegexMatchingEventHandler

from anchore_engine.subsys import logger
import anchore_engine.db.entities.common
from anchore_engine.db.entities.exceptions import TableNotFoundError
from anchore_engine.db.entities.exceptions import is_table_not_found

import anchore_manager.cli.utils

class AnchoreLogWatcher(RegexMatchingEventHandler):
    regexes = [re.compile(".*/anchore-.*\.log$")]
    files = {}

    def do_close(self, event):
        if event.src_path in self.files and self.files[event.src_path]['filehandle']:
            self.files[event.src_path]['filehandle'].close()
        self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

    def on_deleted(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        self.do_close(event)

    def on_modified(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        if not self.files[event.src_path]['filehandle']:
            if os.path.exists(event.src_path):
                self.files[event.src_path]['filehandle'] = open(event.src_path)

        if self.files[event.src_path]['filehandle']:
            patt = re.match(".*anchore-(.*)\.log$", event.src_path)
            if patt:
                logname = patt.group(1)
            else:
                logname = event.src_path

            for line in self.files[event.src_path]['filehandle'].readlines():
                sys.stdout.write("[service:" + str(logname) + "] " + line)

            self.files[event.src_path]['filetell'] = self.files[event.src_path]['filehandle'].tell()

    def on_created(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        if self.files[event.src_path]['filehandle']:
            self.do_close(event)

        if os.path.exists(event.src_path):
            self.files[event.src_path]['filehandle'] = open(event.src_path)
            self.files[event.src_path]['filetell'] = 0

    def on_moved(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}
        self.on_created(event)

    def on_any_event(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

class ServiceThread():

    def __init__(self, thread_target, thread_args):
        self.thread_target = thread_target
        self.thread_args = thread_args
        self.start()

    def start(self):
        self.thread = threading.Thread(target=self.thread_target, args=self.thread_args)
        self.thread.name = self.thread_args[0]
        self.thread.start()

def terminate_service(service, flush_pidfile=False):
    pidfile = "/var/run/" + service + ".pid"
    try:
        thepid = None
        if os.path.exists(pidfile):
            with open(pidfile, 'r') as FH:
                thepid = int(FH.read())

        if thepid:
            print "Found old pid for service: {}. Terminating it".format(service)
            try:
                os.kill(thepid, 0)
            except OSError:
                pass
            else:
                print "killing old twistd pid: " + str(thepid)
                os.kill(thepid, 9)
            if flush_pidfile:
                os.remove(pidfile)
    except Exception as err:
        print "could not shut down running twistd - exception: " + str(err)

def startup_service(service, configdir):
    pidfile = "/var/run/" + service + ".pid"
    logfile = "/var/log/anchore/" + service + ".log"
    # os.environ['ANCHORE_LOGFILE'] = logfile

    print "cleaning up service: " + str(service)
    terminate_service(service, flush_pidfile=True)

    twistd_cmd = '/bin/twistd'
    for f in ['/bin/twistd', '/usr/local/bin/twistd']:
        if os.path.exists(f):
            twistd_cmd = f

    cmd = [twistd_cmd, '--logger=anchore_engine.subsys.twistd_logger.logger', '--pidfile', pidfile, "-n", service, '--config', configdir]
    print "starting service: " + str(service)
    print "\t " + ' '.join(cmd)

    try:
        newenv = os.environ.copy()
        newenv['ANCHORE_LOGFILE'] = logfile
        pipes = subprocess.Popen(cmd, env=newenv)
        sout, serr = pipes.communicate()
        rc = pipes.returncode
        raise Exception("process exited: " + str(rc))
    except Exception as err:
        traceback.print_exc()
        print "service process exited at (" + str(time.ctime()) + "): " + str(err)

    print "exiting service thread"
    return (False)

config = {}
module = None

@click.group(name='service', short_help='Service operations')
@click.pass_obj
def service(ctx_config):
    global config, module
    config = ctx_config

    try:
        # do some DB connection/pre-checks here
        try:

            log_level = 'INFO'
            if config['debug']:
                log_level = 'DEBUG'
            logger.set_log_level(log_level, log_to_stdout=True)

        except Exception as err:
            raise err

    except Exception as err:
        print anchore_manager.cli.utils.format_error_output(config, 'service', {}, err)
        sys.exit(2)

@service.command(name='start', short_help="Start anchore-engine")
@click.option("--auto-upgrade", is_flag=True, help="Perform automatic upgrade on startup")
@click.option("--anchore-module", nargs=1, help="Name of anchore module to call DB routines from (default=anchore_engine)")
@click.option("--skip-config-validate", nargs=1, help="Comma-separated list of configuration file sections to skip specific validation processing (e.g. services,credentials,webhooks)")
def start(auto_upgrade, anchore_module, skip_config_validate):
    global config
    """
    """
    ecode = 0

    auto_upgrade = True

    if not anchore_module:
        module_name = "anchore_engine"
    else:
        module_name = str(anchore_module)

    try:
        service_map = {
            'analyzer': 'anchore-worker',
            'simplequeue': 'anchore-simplequeue',
            'apiext': 'anchore-api',
            'catalog': 'anchore-catalog',
            'kubernetes_webhook': 'anchore-kubernetes-webhook',
            'policy_engine': 'anchore-policy-engine',
            'feeds': 'anchore-feeds'
        }

        validate_params = {
            'services': True,
            'webhooks': True,
            'credentials': True
        }
        if skip_config_validate:
            try:
                items = skip_config_validate.split(',')
                for item in items:
                    validate_params[item] = False
            except Exception as err:
                raise Exception(err)

        # find/set up configuration
        configdir = config['configdir']
        configfile = os.path.join(configdir, "config.yaml")

        localconfig = None
        if os.path.exists(configfile):
            try:
                localconfig = anchore_engine.configuration.localconfig.load_config(configdir=configdir, configfile=configfile, validate_params=validate_params)
            except Exception as err:
                raise Exception("cannot load local configuration: " + str(err))
        else:
            raise Exception("cannot locate configuration file ({})".format(configfile))

        # load the appropriate DB module
        try:
            print "Loading DB routines from module ({})".format(module_name)
            module = importlib.import_module(module_name + ".db.entities.upgrade")
        except TableNotFoundError as ex:
            print "Initialized DB not found."
        except Exception as err:
            raise Exception("Input anchore-module (" + str(module_name) + ") cannot be found/imported - exception: " + str(err))

        # preflight - db checks
        try:
            db_params = anchore_engine.db.entities.common.get_params(localconfig)
            db_params = anchore_manager.cli.utils.init_database(config, db_params['db_connect'], db_params['db_connect_args']['ssl'], db_retries=300)

            in_sync = False
            timed_out = False
            max_timeout = 3600

            timer = time.time()
            while not in_sync and not timed_out:
                code_versions = db_versions = None
                try:
                    code_versions, db_versions = module.get_versions()
                except Exception as err:
                    pass

                if code_versions and db_versions:
                    if code_versions['db_version'] != db_versions['db_version']:
                        if auto_upgrade:
                            print "Performing upgrade."
                            try:
                                # perform the upgrade logic here
                                rc = module.run_upgrade()
                                if rc:
                                    print "Upgrade completed"
                                else:
                                    print "No upgrade necessary. Completed."
                            except Exception as err:
                                raise err

                            in_sync = True
                        else:
                            print "this version of anchore-engine requires the anchore DB version ("+str(code_versions['db_version'])+") but we discovered anchore DB version ("+str(db_versions['db_version'])+") in the running DB - it is safe to run the upgrade while seeing this message - will retry for "+str(max_timeout - int(time.time() - timer))+" more seconds."
                            time.sleep(5)
                    else:
                        print "DB version and code version in sync."
                        in_sync = True
                else:
                    print('Warn: no existing anchore DB data can be discovered, assuming bootstrap')
                    in_sync = True

                if (max_timeout - int(time.time() - timer)) < 0:
                    timed_out = True

            if not in_sync:
                raise Exception("this version of anchore-engine requires the anchore DB version ("+str(code_versions['db_version'])+") but we discovered anchore DB version ("+str(db_versions['db_version'])+") in the running DB - please perform the DB upgrade process and retry")

        except Exception as err:
            raise err

        finally:
            rc = anchore_engine.db.entities.common.do_disconnect()

        # start up the services
        startFailed = False
        if 'ANCHORE_ENGINE_SERVICES' in os.environ:
            input_services = os.environ['ANCHORE_ENGINE_SERVICES'].split()
        else:
            config_services = localconfig.get('services', {})
            if not config_services:
                print('Warn: Did not find any services to execute in the config file')
                sys.exit(1)

            input_services = [ name for name, srv_conf in config_services.items() if srv_conf.get('enabled')]

        services = []
        for service_conf_name in input_services:
            if service_conf_name in service_map.values():
                service = service_conf_name
            else:
                service = service_map.get(service_conf_name)

            if service:
                services.append(service)
            else:
                print('Warn: specified service {} not found in list of available services {} - removing from list of services to start'.format(service_conf_name, service_map.keys()))

        if 'anchore-catalog' in services:
            services.remove('anchore-catalog')
            services.insert(0, 'anchore-catalog')

        if not services:
            print "No services found in ANCHORE_ENGINE_SERVICES or as enabled in config.yaml to start - exiting"
            sys.exit(1)

        print('Starting services: {}'.format(services))
        #services = ['anchore-catalog', 'anchore-api', 'anchore-simplequeue', 'anchore-worker', 'anchore-kubernetes-webhook', 'anchore-policy-engine']

        try:
            if not os.path.exists("/var/log/anchore"):
                os.makedirs("/var/log/anchore/", 0755)
        except Exception as err:
            print "cannot create log directory /var/log/anchore - exception: " + str(err)
            raise err

        pids = []
        keepalive_threads = []
        for service in services:
            pidfile = "/var/run/" + service + ".pid"
            try:
                service_thread = ServiceThread(startup_service, (service, configdir))
                keepalive_threads.append(service_thread)
                max_tries = 30
                tries = 0
                while not os.path.exists(pidfile) and tries < max_tries:
                    time.sleep(1)
                    tries = tries + 1

                time.sleep(2)
            except Exception as err:
                startFailed = True
                print "service start failed - exception: " + str(err)

        if startFailed:
            print "one or more services failed to start. cleanly terminating the others"
            for service in services:
                terminate_service(service, flush_pidfile=True)

            sys.exit(1)
        else:
            # start up the log watchers
            try:
                observer = Observer()
                observer.schedule(AnchoreLogWatcher(), path="/var/log/anchore/")
                observer.start()

                try:
                    while True:
                        time.sleep(1)
                        if 'auto_restart_services' in localconfig and localconfig['auto_restart_services']:
                            for service_thread in keepalive_threads:
                                if not service_thread.thread.is_alive():
                                    print "restarting service " + service_thread.thread.name
                                    service_thread.start()

                except KeyboardInterrupt:
                    observer.stop()
                observer.join()

            except Exception as err:
                print "failed to startup log watchers - exception: " + str(err)
                raise err

        print "THERE"
    except Exception as err:
        print anchore_manager.cli.utils.format_error_output(config, 'servicestart', {}, err)
        if not ecode:
            ecode = 2

    anchore_manager.cli.utils.doexit(ecode)
<commit_msg>auto-upgrade message updated<commit_after>import os
import re
import sys
import json
import time
import click
import importlib
import traceback
import threading
import subprocess
import watchdog
import anchore_engine.configuration.localconfig
#import anchore_engine.db.entities.upgrade

from watchdog.observers import Observer
from watchdog.events import RegexMatchingEventHandler

from anchore_engine.subsys import logger
import anchore_engine.db.entities.common
from anchore_engine.db.entities.exceptions import TableNotFoundError
from anchore_engine.db.entities.exceptions import is_table_not_found

import anchore_manager.cli.utils

class AnchoreLogWatcher(RegexMatchingEventHandler):
    regexes = [re.compile(".*/anchore-.*\.log$")]
    files = {}

    def do_close(self, event):
        if event.src_path in self.files and self.files[event.src_path]['filehandle']:
            self.files[event.src_path]['filehandle'].close()
        self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

    def on_deleted(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        self.do_close(event)

    def on_modified(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        if not self.files[event.src_path]['filehandle']:
            if os.path.exists(event.src_path):
                self.files[event.src_path]['filehandle'] = open(event.src_path)

        if self.files[event.src_path]['filehandle']:
            patt = re.match(".*anchore-(.*)\.log$", event.src_path)
            if patt:
                logname = patt.group(1)
            else:
                logname = event.src_path

            for line in self.files[event.src_path]['filehandle'].readlines():
                sys.stdout.write("[service:" + str(logname) + "] " + line)

            self.files[event.src_path]['filetell'] = self.files[event.src_path]['filehandle'].tell()

    def on_created(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

        if self.files[event.src_path]['filehandle']:
            self.do_close(event)

        if os.path.exists(event.src_path):
            self.files[event.src_path]['filehandle'] = open(event.src_path)
            self.files[event.src_path]['filetell'] = 0

    def on_moved(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}
        self.on_created(event)

    def on_any_event(self, event):
        if event.src_path not in self.files:
            self.files[event.src_path] = {'filehandle': None, 'filetell': 0}

class ServiceThread():

    def __init__(self, thread_target, thread_args):
        self.thread_target = thread_target
        self.thread_args = thread_args
        self.start()

    def start(self):
        self.thread = threading.Thread(target=self.thread_target, args=self.thread_args)
        self.thread.name = self.thread_args[0]
        self.thread.start()

def terminate_service(service, flush_pidfile=False):
    pidfile = "/var/run/" + service + ".pid"
    try:
        thepid = None
        if os.path.exists(pidfile):
            with open(pidfile, 'r') as FH:
                thepid = int(FH.read())

        if thepid:
            print "Found old pid for service: {}. Terminating it".format(service)
            try:
                os.kill(thepid, 0)
            except OSError:
                pass
            else:
                print "killing old twistd pid: " + str(thepid)
                os.kill(thepid, 9)
            if flush_pidfile:
                os.remove(pidfile)
    except Exception as err:
        print "could not shut down running twistd - exception: " + str(err)

def startup_service(service, configdir):
    pidfile = "/var/run/" + service + ".pid"
    logfile = "/var/log/anchore/" + service + ".log"
    # os.environ['ANCHORE_LOGFILE'] = logfile

    print "cleaning up service: " + str(service)
    terminate_service(service, flush_pidfile=True)

    twistd_cmd = '/bin/twistd'
    for f in ['/bin/twistd', '/usr/local/bin/twistd']:
        if os.path.exists(f):
            twistd_cmd = f

    cmd = [twistd_cmd, '--logger=anchore_engine.subsys.twistd_logger.logger', '--pidfile', pidfile, "-n", service, '--config', configdir]
    print "starting service: " + str(service)
    print "\t " + ' '.join(cmd)

    try:
        newenv = os.environ.copy()
        newenv['ANCHORE_LOGFILE'] = logfile
        pipes = subprocess.Popen(cmd, env=newenv)
        sout, serr = pipes.communicate()
        rc = pipes.returncode
        raise Exception("process exited: " + str(rc))
    except Exception as err:
        traceback.print_exc()
        print "service process exited at (" + str(time.ctime()) + "): " + str(err)

    print "exiting service thread"
    return (False)

config = {}
module = None

@click.group(name='service', short_help='Service operations')
@click.pass_obj
def service(ctx_config):
    global config, module
    config = ctx_config

    try:
        # do some DB connection/pre-checks here
        try:

            log_level = 'INFO'
            if config['debug']:
                log_level = 'DEBUG'
            logger.set_log_level(log_level, log_to_stdout=True)

        except Exception as err:
            raise err

    except Exception as err:
        print anchore_manager.cli.utils.format_error_output(config, 'service', {}, err)
        sys.exit(2)

@service.command(name='start', short_help="Start anchore-engine")
@click.option("--auto-upgrade", is_flag=True, help="Perform automatic upgrade on startup")
@click.option("--anchore-module", nargs=1, help="Name of anchore module to call DB routines from (default=anchore_engine)")
@click.option("--skip-config-validate", nargs=1, help="Comma-separated list of configuration file sections to skip specific validation processing (e.g. services,credentials,webhooks)")
def start(auto_upgrade, anchore_module, skip_config_validate):
    global config
    """
    """
    ecode = 0

    auto_upgrade = True

    if not anchore_module:
        module_name = "anchore_engine"
    else:
        module_name = str(anchore_module)

    try:
        service_map = {
            'analyzer': 'anchore-worker',
            'simplequeue': 'anchore-simplequeue',
            'apiext': 'anchore-api',
            'catalog': 'anchore-catalog',
            'kubernetes_webhook': 'anchore-kubernetes-webhook',
            'policy_engine': 'anchore-policy-engine',
            'feeds': 'anchore-feeds'
        }

        validate_params = {
            'services': True,
            'webhooks': True,
            'credentials': True
        }
        if skip_config_validate:
            try:
                items = skip_config_validate.split(',')
                for item in items:
                    validate_params[item] = False
            except Exception as err:
                raise Exception(err)

        # find/set up configuration
        configdir = config['configdir']
        configfile = os.path.join(configdir, "config.yaml")

        localconfig = None
        if os.path.exists(configfile):
            try:
                localconfig = anchore_engine.configuration.localconfig.load_config(configdir=configdir, configfile=configfile, validate_params=validate_params)
            except Exception as err:
                raise Exception("cannot load local configuration: " + str(err))
        else:
            raise Exception("cannot locate configuration file ({})".format(configfile))

        # load the appropriate DB module
        try:
            print "Loading DB routines from module ({})".format(module_name)
            module = importlib.import_module(module_name + ".db.entities.upgrade")
        except TableNotFoundError as ex:
            print "Initialized DB not found."
        except Exception as err:
            raise Exception("Input anchore-module (" + str(module_name) + ") cannot be found/imported - exception: " + str(err))

        # preflight - db checks
        try:
            db_params = anchore_engine.db.entities.common.get_params(localconfig)
            db_params = anchore_manager.cli.utils.init_database(config, db_params['db_connect'], db_params['db_connect_args']['ssl'], db_retries=300)

            in_sync = False
            timed_out = False
            max_timeout = 3600

            timer = time.time()
            while not in_sync and not timed_out:
                code_versions = db_versions = None
                try:
                    code_versions, db_versions = module.get_versions()
                except Exception as err:
                    pass

                if code_versions and db_versions:
                    if code_versions['db_version'] != db_versions['db_version']:
                        if auto_upgrade:
                            print "Auto-upgrade is set - performing upgrade."
                            try:
                                # perform the upgrade logic here
                                rc = module.run_upgrade()
                                if rc:
                                    print "Upgrade completed"
                                else:
                                    print "No upgrade necessary. Completed."
                            except Exception as err:
                                raise err

                            in_sync = True
                        else:
                            print "this version of anchore-engine requires the anchore DB version ("+str(code_versions['db_version'])+") but we discovered anchore DB version ("+str(db_versions['db_version'])+") in the running DB - it is safe to run the upgrade while seeing this message - will retry for "+str(max_timeout - int(time.time() - timer))+" more seconds."
                            time.sleep(5)
                    else:
                        print "DB version and code version in sync."
                        in_sync = True
                else:
                    print('Warn: no existing anchore DB data can be discovered, assuming bootstrap')
                    in_sync = True

                if (max_timeout - int(time.time() - timer)) < 0:
                    timed_out = True

            if not in_sync:
                raise Exception("this version of anchore-engine requires the anchore DB version ("+str(code_versions['db_version'])+") but we discovered anchore DB version ("+str(db_versions['db_version'])+") in the running DB - please perform the DB upgrade process and retry")

        except Exception as err:
            raise err

        finally:
            rc = anchore_engine.db.entities.common.do_disconnect()

        # start up the services
        startFailed = False
        if 'ANCHORE_ENGINE_SERVICES' in os.environ:
            input_services = os.environ['ANCHORE_ENGINE_SERVICES'].split()
        else:
            config_services = localconfig.get('services', {})
            if not config_services:
                print('Warn: Did not find any services to execute in the config file')
                sys.exit(1)

            input_services = [ name for name, srv_conf in config_services.items() if srv_conf.get('enabled')]

        services = []
        for service_conf_name in input_services:
            if service_conf_name in service_map.values():
                service = service_conf_name
            else:
                service = service_map.get(service_conf_name)

            if service:
                services.append(service)
            else:
                print('Warn: specified service {} not found in list of available services {} - removing from list of services to start'.format(service_conf_name, service_map.keys()))

        if 'anchore-catalog' in services:
            services.remove('anchore-catalog')
            services.insert(0, 'anchore-catalog')

        if not services:
            print "No services found in ANCHORE_ENGINE_SERVICES or as enabled in config.yaml to start - exiting"
            sys.exit(1)

        print('Starting services: {}'.format(services))
        #services = ['anchore-catalog', 'anchore-api', 'anchore-simplequeue', 'anchore-worker', 'anchore-kubernetes-webhook', 'anchore-policy-engine']

        try:
            if not os.path.exists("/var/log/anchore"):
                os.makedirs("/var/log/anchore/", 0755)
        except Exception as err:
            print "cannot create log directory /var/log/anchore - exception: " + str(err)
            raise err

        pids = []
        keepalive_threads = []
        for service in services:
            pidfile = "/var/run/" + service + ".pid"
            try:
                service_thread = ServiceThread(startup_service, (service, configdir))
                keepalive_threads.append(service_thread)
                max_tries = 30
                tries = 0
                while not os.path.exists(pidfile) and tries < max_tries:
                    time.sleep(1)
                    tries = tries + 1

                time.sleep(2)
            except Exception as err:
                startFailed = True
                print "service start failed - exception: " + str(err)

        if startFailed:
            print "one or more services failed to start. cleanly terminating the others"
            for service in services:
                terminate_service(service, flush_pidfile=True)

            sys.exit(1)
        else:
            # start up the log watchers
            try:
                observer = Observer()
                observer.schedule(AnchoreLogWatcher(), path="/var/log/anchore/")
                observer.start()

                try:
                    while True:
                        time.sleep(1)
                        if 'auto_restart_services' in localconfig and localconfig['auto_restart_services']:
                            for service_thread in keepalive_threads:
                                if not service_thread.thread.is_alive():
                                    print "restarting service " + service_thread.thread.name
                                    service_thread.start()

                except KeyboardInterrupt:
                    observer.stop()
                observer.join()

            except Exception as err:
                print "failed to startup log watchers - exception: " + str(err)
                raise err

        print "THERE"
    except Exception as err:
        print anchore_manager.cli.utils.format_error_output(config, 'servicestart', {}, err)
        if not ecode:
            ecode = 2

    anchore_manager.cli.utils.doexit(ecode)
<|endoftext|>


<commit_before><commit_msg>add tests for new email blocking functionality<commit_after><|endoftext|>


<commit_before>import idiokit
from abusehelper.core import bot, taskfarm

class RoomBot(bot.ServiceBot):
    def __init__(self, *args, **keys):
        bot.ServiceBot.__init__(self, *args, **keys)
        self.room_handlers = taskfarm.TaskFarm(self._handle_room)

    @idiokit.stream
    def _handle_room(self, name):
        msg = "room {0!r}".format(name)
        attrs = events.Event(type="room", service=self.bot_name, room=name)

        with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log:
            log.open("Joining " + msg, attrs, status="joining")
            room = yield self.xmpp.muc.join(name, self.bot_name)

            log.open("Joined " + msg, attrs, status="joined")
            try:
                yield room
            finally:
                log.close("Left " + msg, attrs, status="left")

    def to_room(self, name):
        return self.room_handlers.inc(name) | idiokit.consume()

    def from_room(self, name):
        return idiokit.consume() | self.room_handlers.inc(name)

import time
import collections
from abusehelper.core import events

class WindowBot(RoomBot):
    @idiokit.stream
    def process(self, ids, queue, window_time):
        while True:
            event = yield idiokit.next()

            current_time = time.time()
            expire_time = current_time + window_time

            eid = events.hexdigest(event)

            event.add("id", eid)
            yield idiokit.send(event)

            ids[eid] = expire_time
            queue.append((expire_time, eid))

    @idiokit.stream
    def purge(self, ids, queue):
        while True:
            yield idiokit.sleep(1.0)

            current_time = time.time()

            while queue and queue[0][0] <= current_time:
                expire_time, eid = queue.popleft()
                if ids.get(eid, None) != expire_time:
                    continue

                del ids[eid]

                event = events.Event()
                event.add("id", eid)
                yield idiokit.send(event)

    @idiokit.stream
    def session(self, state, src_room, dst_room, window_time=60.0):
        queue = collections.deque()
        ids = dict()

        to = self.to_room(dst_room)
        idiokit.pipe(self.purge(ids, queue),
                     events.events_to_elements(),
                     to)

        yield (self.from_room(src_room)
               | events.stanzas_to_events()
               | self.process(ids, queue, window_time)
               | events.events_to_elements()
               | to)

if __name__ == "__main__":
    WindowBot.from_command_line().execute()
<commit_msg>abusehelper.contrib.windowbot: Use the new-style "id:open"/"id:close" scheme instead of the legacy "id" scheme for span events.<commit_after>import idiokit
from abusehelper.core import bot, taskfarm

class RoomBot(bot.ServiceBot):
    def __init__(self, *args, **keys):
        bot.ServiceBot.__init__(self, *args, **keys)
        self.room_handlers = taskfarm.TaskFarm(self._handle_room)

    @idiokit.stream
    def _handle_room(self, name):
        msg = "room {0!r}".format(name)
        attrs = events.Event(type="room", service=self.bot_name, room=name)

        with self.log.stateful(repr(self.xmpp.jid), "room", repr(name)) as log:
            log.open("Joining " + msg, attrs, status="joining")
            room = yield self.xmpp.muc.join(name, self.bot_name)

            log.open("Joined " + msg, attrs, status="joined")
            try:
                yield room
            finally:
                log.close("Left " + msg, attrs, status="left")

    def to_room(self, name):
        return self.room_handlers.inc(name) | idiokit.consume()

    def from_room(self, name):
        return idiokit.consume() | self.room_handlers.inc(name)

import time
import collections
from abusehelper.core import events

class WindowBot(RoomBot):
    @idiokit.stream
    def process(self, ids, queue, window_time):
        while True:
            event = yield idiokit.next()

            current_time = time.time()
            expire_time = current_time + window_time

            eid = events.hexdigest(event)
            count, items = ids.get(eid, (0, event.items()))
            ids[eid] = count + 1, items

            if count == 0:
                yield idiokit.send(event.union({
                    "id:open": eid
                }))
            queue.append((expire_time, eid))

    @idiokit.stream
    def purge(self, ids, queue):
        while True:
            yield idiokit.sleep(1.0)

            current_time = time.time()

            while queue and queue[0][0] <= current_time:
                expire_time, eid = queue.popleft()

                count, items = ids.pop(eid)
                if count > 1:
                    ids[eid] = count - 1, items
                else:
                    yield idiokit.send(events.Event(items).union({
                        "id:close": eid
                    }))

    @idiokit.stream
    def session(self, state, src_room, dst_room, window_time=60.0):
        queue = collections.deque()
        ids = dict()

        to = self.to_room(dst_room)
        idiokit.pipe(self.purge(ids, queue),
                     events.events_to_elements(),
                     to)

        yield (self.from_room(src_room)
               | events.stanzas_to_events()
               | self.process(ids, queue, window_time)
               | events.events_to_elements()
               | to)

if __name__ == "__main__":
    WindowBot.from_command_line().execute()
<|endoftext|>


<commit_before>"""
MIT License

Copyright (c) 2017 cgalleguillosm

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from accasim.utils.reader_class import workload_parser_base
from accasim.utils.file import plain_file_reader
from accasim.utils.misc import load_config, type_regexp, DEFAULT_SIMULATION
from re import compile as _compile

def define_result_parser(simulator_config=None):
    """

    :return:
    """
    try:
        if simulator_config is not None:
            _schedule_output = load_config(simulator_config)['schedule_output']
        else:
            # If no simulation config is supplied, DEFAULT_SIMULATION is used
            _schedule_output = DEFAULT_SIMULATION().parameters['SCHEDULE_OUTPUT']
        # _separators = _schedule_output['separators']
        _format = _schedule_output['format']
        _attributes = _schedule_output['attributes']
    except KeyError as e:
        print(
            'Schedule output format not identified. Please check the simulator configuration file for the key \'schedule_output\'.')

    for _attr_name, _data_type in _attributes.items():
        _format = _format.replace('{' + _attr_name + '}', type_regexp(_data_type[-1]).format(_attr_name))
    return schedule_parser(_format, [])

class schedule_parser(workload_parser_base):
    def __init__(self, regexp, updater=[]):
        """
        schedule_parser class is an implementation of the :class:`accasim.utils.reader_class.workload_parser_base`

        :param regexp:

        """
        workload_parser_base.__init__(self)
        self.regexp = _compile(regexp)
        self.updater = updater

    def parse_line(self, line):
        """

        :param line:
        :return:
        """
        _matches = self.regexp.match(line)
        if not _matches:
            return None
        _dict = _matches.groupdict()
        for u in self.updater:
            u(_dict)
        return _dict

class workload_file_reader:

    def __init__(self, workload, reg_exp, tweak_class, updater=[]):
        """

        :param workload:
        :param reg_exp:
        :param tweak_class:
        :param updater:
        """
        self.reader = plain_file_reader(workload)
        self.parser = schedule_parser(reg_exp, updater)
        self.tweak = tweak_class

    def next(self, omit_startwith=';'):
        """

        :param omit_startwith:
        :return:
        """
        if self.reader.EOF:
            return None
        line = self.reader.nextline()
        if not line or line.isspace():
            return None
        try:
            while omit_startwith in line:
                line = self.reader.nextline()
                if not line:
                    return None
        except TypeError as e:
            print('Error line: {}'.format(line))
            print(e)
            exit()
        parsed_line = self.parser.parse_line(line)
        return self.tweak.tweak_function(parsed_line)
<commit_msg>Exit on fail<commit_after>"""
MIT License

Copyright (c) 2017 cgalleguillosm

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from accasim.utils.reader_class import workload_parser_base
from accasim.utils.file import plain_file_reader
from accasim.utils.misc import load_config, type_regexp, DEFAULT_SIMULATION
from re import compile as _compile

def define_result_parser(simulator_config=None):
    """

    :return:
    """
    try:
        if simulator_config is not None:
            _schedule_output = load_config(simulator_config)['schedule_output']
        else:
            # If no simulation config is supplied, DEFAULT_SIMULATION is used
            _schedule_output = DEFAULT_SIMULATION().parameters['SCHEDULE_OUTPUT']
        # _separators = _schedule_output['separators']
        _format = _schedule_output['format']
        _attributes = _schedule_output['attributes']
    except KeyError as e:
        print(
            'Schedule output format not identified. Please check the simulator configuration file for the key \'schedule_output\'.')
        exit()

    for _attr_name, _data_type in _attributes.items():
        _format = _format.replace('{' + _attr_name + '}', type_regexp(_data_type[-1]).format(_attr_name))
    return schedule_parser(_format, [])

class schedule_parser(workload_parser_base):
    def __init__(self, regexp, updater=[]):
        """
        schedule_parser class is an implementation of the :class:`accasim.utils.reader_class.workload_parser_base`

        :param regexp:

        """
        workload_parser_base.__init__(self)
        self.regexp = _compile(regexp)
        self.updater = updater

    def parse_line(self, line):
        """

        :param line:
        :return:
        """
        _matches = self.regexp.match(line)
        if not _matches:
            return None
        _dict = _matches.groupdict()
        for u in self.updater:
            u(_dict)
        return _dict

class workload_file_reader:

    def __init__(self, workload, reg_exp, tweak_class, updater=[]):
        """

        :param workload:
        :param reg_exp:
        :param tweak_class:
        :param updater:
        """
        self.reader = plain_file_reader(workload)
        self.parser = schedule_parser(reg_exp, updater)
        self.tweak = tweak_class

    def next(self, omit_startwith=';'):
        """

        :param omit_startwith:
        :return:
        """
        if self.reader.EOF:
            return None
        line = self.reader.nextline()
        if not line or line.isspace():
            return None
        try:
            while omit_startwith in line:
                line = self.reader.nextline()
                if not line:
                    return None
        except TypeError as e:
            print('Error line: {}'.format(line))
            print(e)
            exit()
        parsed_line = self.parser.parse_line(line)
        return self.tweak.tweak_function(parsed_line)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Joel Grand-Guillaume
#    Copyright 2011-2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import sys
import traceback

import psycopg2

from openerp.tools.translate import _
import datetime
from openerp.osv.orm import Model
from openerp.osv import fields, osv
from parser import new_bank_statement_parser

class AccountStatementProfil(Model):
    _inherit = "account.statement.profile"

    def get_import_type_selection(self, cr, uid, context=None):
        """
        Has to be inherited to add parser
        """
        return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')]

    _columns = {
        'launch_import_completion': fields.boolean(
            "Launch completion after import",
            help="Tic that box to automatically launch the completion "
                 "on each imported file using this profile."),
        'last_import_date': fields.datetime("Last Import Date"),
        #  we remove deprecated as it floods logs in standard/warning level sob...
        'rec_log': fields.text('log', readonly=True), #  Deprecated
        'import_type': fields.selection(
            get_import_type_selection,
            'Type of import',
            required=True,
            help="Choose here the method by which you want to import bank"
                 "statement for this profile."),

    }

    def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, context):
        """
        Write the log in the logger

        :param int/long statement_id: ID of the concerned account.bank.statement
        :param int/long num_lines: Number of line that have been parsed
        :return: True
        """
        self.message_post(cr,
                          uid,
                          ids,
                          body=_('Statement ID %s have been imported with %s lines.') %
                                (statement_id, num_lines),
                          context=context)
        return True

    def prepare_global_commission_line_vals(
            self, cr, uid, parser, result_row_list, profile, statement_id, context):
        """
        Prepare the global commission line if there is one. The global
        commission is computed by by calling the get_st_line_commision
        of the parser. Feel free to override the method to compute
        your own commission line from the result_row_list.

            :param:    browse_record of the current parser
            :param:    result_row_list: [{'key':value}]
            :param:    profile: browserecord of account.statement.profile
            :param:    statement_id : int/long of the current importing statement ID
            :param:    context: global context
            return:    dict of vals that will be passed to create method of statement line.
        """
        comm_values = False
        if parser.get_st_line_commision():
            partner_id = profile.partner_id and profile.partner_id.id or False
            commission_account_id = profile.commission_account_id and profile.commission_account_id.id or False
            commission_analytic_id = profile.commission_analytic_id and profile.commission_analytic_id.id or False
            comm_values = {
                'name': 'IN ' + _('Commission line'),
                'date': datetime.datetime.now().date(),
                'amount': parser.get_st_line_commision(),
                'partner_id': partner_id,
                'type': 'general',
                'statement_id': statement_id,
                'account_id': commission_account_id,
                'ref': 'commission',
                'analytic_account_id': commission_analytic_id,
                # !! We set the already_completed so auto-completion will not update those values !
                'already_completed': True,
            }
        return comm_values

    def prepare_statetement_lines_vals(
            self, cr, uid, parser_vals, account_payable, account_receivable,
            statement_id, context):
        """
        Hook to build the values of a line from the parser returned values. At
        least it fullfill the statement_id and account_id. Overide it to add your
        own completion if needed.

        :param dict of vals from parser for account.bank.statement.line (called by
                parser.get_st_line_vals)
        :param int/long account_payable: ID of the receivable account to use
        :param int/long account_receivable: ID of the payable account to use
        :param int/long statement_id: ID of the concerned account.bank.statement
        :return : dict of vals that will be passed to create method of statement line.
        """
        statement_obj = self.pool.get('account.bank.statement')
        values = parser_vals
        values['statement_id'] = statement_id
        values['account_id'] = statement_obj.get_account_for_counterpart(cr,
                                                                         uid,
                                                                         parser_vals['amount'],
                                                                         account_receivable,
                                                                         account_payable)

        date = values.get('date')
        period_memoizer = context.get('period_memoizer')
        if not period_memoizer:
            period_memoizer = {}
            context['period_memoizer'] = period_memoizer
        if period_memoizer.get(date):
            values['period_id'] = period_memoizer[date]
        else:
            # This is awfully slow...
            periods = self.pool.get('account.period').find(cr, uid,
                                                           dt=values.get('date'),
                                                           context=context)
            values['period_id'] = periods[0]
            period_memoizer[date] = periods[0]
        values['type'] = 'general'
        return values

    def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None):
        """
        Create a bank statement with the given profile and parser. It will fullfill the bank statement
        with the values of the file providen, but will not complete data (like finding the partner, or
        the right account). This will be done in a second step with the completion rules.
        It will also create the commission line if it apply and record the providen file as
        an attachement of the bank statement.

        :param int/long profile_id: ID of the profile used to import the file
        :param filebuffer file_stream: binary of the providen file
        :param char: ftype represent the file exstension (csv by default)
        :return: ID of the created account.bank.statemênt
        """
        statement_obj = self.pool.get('account.bank.statement')
        statement_line_obj = self.pool.get('account.bank.statement.line')
        attachment_obj = self.pool.get('ir.attachment')
        prof_obj = self.pool.get("account.statement.profile")
        if not profile_id:
            raise osv.except_osv(_("No Profile !"),
                                 _("You must provide a valid profile to import a bank statement !"))
        prof = prof_obj.browse(cr, uid, profile_id, context=context)

        parser = new_bank_statement_parser(prof.import_type, ftype=ftype)
        result_row_list = parser.parse(file_stream)
        # Check all key are present in account.bank.statement.line !!
        if not result_row_list:
            raise osv.except_osv(_("Nothing to import"),
                                 _("The file is empty"))
        parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys()
        for col in parsed_cols:
            if col not in statement_line_obj._columns:
                raise osv.except_osv(_("Missing column !"),
                                     _("Column %s you try to import is not "
                                       "present in the bank statement line !") % col)

        statement_id = statement_obj.create(cr, uid,
                                            {'profile_id': prof.id},
                                            context=context)
        if prof.receivable_account_id:
            account_receivable, account_payable = prof.receivable_account_id.id
        else:
            account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(
                                                       cr, uid, context)
        try:
            # Record every line in the bank statement and compute the global commission
            # based on the commission_amount column
            statement_store = []
            for line in result_row_list:
                parser_vals = parser.get_st_line_vals(line)
                values = self.prepare_statetement_lines_vals(cr, uid, parser_vals, account_payable,
                                                             account_receivable, statement_id, context)
                statement_store.append(values)
            # Hack to bypass ORM poor perfomance. Sob...
            statement_line_obj._insert_lines(cr, uid, statement_store, context=context)

            # Build and create the global commission line for the whole statement
            comm_vals = self.prepare_global_commission_line_vals(cr, uid, parser, result_row_list,
                                                                 prof, statement_id, context)
            if comm_vals:
                statement_line_obj.create(cr, uid, comm_vals, context=context)
            else:
                # Trigger store field computation if someone has better idea
                start_bal = statement_obj.read(cr, uid, statement_id,
                                               ['balance_start'],
                                               context=context)
                start_bal = start_bal['balance_start']
                statement_obj.write(cr, uid, [statement_id],
                                    {'balance_start': start_bal})

            attachment_obj.create(cr,
                                  uid,
                                  {'name': 'statement file',
                                   'datas': file_stream,
                                   'datas_fname': "%s.%s" % (
                                       datetime.datetime.now().date(),
                                       ftype),
                                   'res_model': 'account.bank.statement',
                                   'res_id': statement_id},
                                  context=context)

            # If user ask to launch completion at end of import, do it !
            if prof.launch_import_completion:
                statement_obj.button_auto_completion(cr, uid, [statement_id], context)

            # Write the needed log infos on profile
            self.write_logs_after_import(cr, uid, prof.id,
                                         statement_id,
                                         len(result_row_list),
                                         context)

        except Exception:
            statement_obj.unlink(cr, uid, [statement_id], context=context)
            error_type, error_value, trbk = sys.exc_info()
            st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value)
            st += ''.join(traceback.format_tb(trbk, 30))
            raise osv.except_osv(_("Statement import error"),
                                 _("The statement cannot be created : %s") % st)
        return statement_id

class AccountStatementLine(Model):
    """
    Add sparse field on the statement line to allow to store all the
    bank infos that are given by an office. In this basic sample case
    it concern only commission_amount.
    """
    _inherit = "account.bank.statement.line"

    def _get_available_columns(self, statement_store):
        """Return writeable by SQL columns"""
        statement_line_obj = self.pool['account.bank.statement.line']
        model_cols = statement_line_obj._columns
        avail = [k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct')]
        keys = [k for k in statement_store[0].keys() if k in avail]
        keys.sort()
        return keys

    def _insert_lines(self, cr, uid, statement_store, context=None):
        """ Do raw insert into database because ORM is awfully slow
            when doing batch write. It is a shame that batch function
            does not exist"""
        statement_line_obj = self.pool['account.bank.statement.line']
        statement_line_obj.check_access_rule(cr, uid, [], 'create')
        statement_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
        cols = self._get_available_columns(statement_store)
        tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
        sql = "INSERT INTO account_bank_statement_line (%s) VALUES (%s);" % tmp_vals
        try:
            cr.executemany(sql, tuple(statement_store))
        except psycopg2.Error as sql_err:
            cr.rollback()
            raise osv.except_osv(_("ORM bypass error"),
                                 sql_err.pgerror)

    def _update_line(self, cr, uid, vals, context=None):
        """ Do raw update into database because ORM is awfully slow
            when doing batch write. It is a shame that batch function
            does not exist"""
        cols = self._get_available_columns([vals])
        tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols]))
        sql = "UPDATE account_bank_statement_line SET %s where id = %%(id)s;" % tmp_vals
        try:
            cr.execute(sql, vals)
        except psycopg2.Error as sql_err:
            cr.rollback()
            raise osv.except_osv(_("ORM bypass error"),
                                 sql_err.pgerror)

    _columns = {
        'commission_amount': fields.sparse(
            type='float',
            string='Line Commission Amount',
            serialization_field='additionnal_bank_fields'),
    }
<commit_msg>[FIX] assignement<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Joel Grand-Guillaume
#    Copyright 2011-2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import sys
import traceback

import psycopg2

from openerp.tools.translate import _
import datetime
from openerp.osv.orm import Model
from openerp.osv import fields, osv
from parser import new_bank_statement_parser

class AccountStatementProfil(Model):
    _inherit = "account.statement.profile"

    def get_import_type_selection(self, cr, uid, context=None):
        """
        Has to be inherited to add parser
        """
        return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')]

    _columns = {
        'launch_import_completion': fields.boolean(
            "Launch completion after import",
            help="Tic that box to automatically launch the completion "
                 "on each imported file using this profile."),
        'last_import_date': fields.datetime("Last Import Date"),
        #  we remove deprecated as it floods logs in standard/warning level sob...
        'rec_log': fields.text('log', readonly=True), #  Deprecated
        'import_type': fields.selection(
            get_import_type_selection,
            'Type of import',
            required=True,
            help="Choose here the method by which you want to import bank"
                 "statement for this profile."),

    }

    def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, context):
        """
        Write the log in the logger

        :param int/long statement_id: ID of the concerned account.bank.statement
        :param int/long num_lines: Number of line that have been parsed
        :return: True
        """
        self.message_post(cr,
                          uid,
                          ids,
                          body=_('Statement ID %s have been imported with %s lines.') %
                                (statement_id, num_lines),
                          context=context)
        return True

    def prepare_global_commission_line_vals(
            self, cr, uid, parser, result_row_list, profile, statement_id, context):
        """
        Prepare the global commission line if there is one. The global
        commission is computed by by calling the get_st_line_commision
        of the parser. Feel free to override the method to compute
        your own commission line from the result_row_list.

            :param:    browse_record of the current parser
            :param:    result_row_list: [{'key':value}]
            :param:    profile: browserecord of account.statement.profile
            :param:    statement_id : int/long of the current importing statement ID
            :param:    context: global context
            return:    dict of vals that will be passed to create method of statement line.
        """
        comm_values = False
        if parser.get_st_line_commision():
            partner_id = profile.partner_id and profile.partner_id.id or False
            commission_account_id = profile.commission_account_id and profile.commission_account_id.id or False
            commission_analytic_id = profile.commission_analytic_id and profile.commission_analytic_id.id or False
            comm_values = {
                'name': 'IN ' + _('Commission line'),
                'date': datetime.datetime.now().date(),
                'amount': parser.get_st_line_commision(),
                'partner_id': partner_id,
                'type': 'general',
                'statement_id': statement_id,
                'account_id': commission_account_id,
                'ref': 'commission',
                'analytic_account_id': commission_analytic_id,
                # !! We set the already_completed so auto-completion will not update those values !
                'already_completed': True,
            }
        return comm_values

    def prepare_statetement_lines_vals(
            self, cr, uid, parser_vals, account_payable, account_receivable,
            statement_id, context):
        """
        Hook to build the values of a line from the parser returned values. At
        least it fullfill the statement_id and account_id. Overide it to add your
        own completion if needed.

        :param dict of vals from parser for account.bank.statement.line (called by
                parser.get_st_line_vals)
        :param int/long account_payable: ID of the receivable account to use
        :param int/long account_receivable: ID of the payable account to use
        :param int/long statement_id: ID of the concerned account.bank.statement
        :return : dict of vals that will be passed to create method of statement line.
        """
        statement_obj = self.pool.get('account.bank.statement')
        values = parser_vals
        values['statement_id'] = statement_id
        values['account_id'] = statement_obj.get_account_for_counterpart(cr,
                                                                         uid,
                                                                         parser_vals['amount'],
                                                                         account_receivable,
                                                                         account_payable)

        date = values.get('date')
        period_memoizer = context.get('period_memoizer')
        if not period_memoizer:
            period_memoizer = {}
            context['period_memoizer'] = period_memoizer
        if period_memoizer.get(date):
            values['period_id'] = period_memoizer[date]
        else:
            # This is awfully slow...
            periods = self.pool.get('account.period').find(cr, uid,
                                                           dt=values.get('date'),
                                                           context=context)
            values['period_id'] = periods[0]
            period_memoizer[date] = periods[0]
        values['type'] = 'general'
        return values

    def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None):
        """
        Create a bank statement with the given profile and parser. It will fullfill the bank statement
        with the values of the file providen, but will not complete data (like finding the partner, or
        the right account). This will be done in a second step with the completion rules.
        It will also create the commission line if it apply and record the providen file as
        an attachement of the bank statement.

        :param int/long profile_id: ID of the profile used to import the file
        :param filebuffer file_stream: binary of the providen file
        :param char: ftype represent the file exstension (csv by default)
        :return: ID of the created account.bank.statemênt
        """
        statement_obj = self.pool.get('account.bank.statement')
        statement_line_obj = self.pool.get('account.bank.statement.line')
        attachment_obj = self.pool.get('ir.attachment')
        prof_obj = self.pool.get("account.statement.profile")
        if not profile_id:
            raise osv.except_osv(_("No Profile !"),
                                 _("You must provide a valid profile to import a bank statement !"))
        prof = prof_obj.browse(cr, uid, profile_id, context=context)

        parser = new_bank_statement_parser(prof.import_type, ftype=ftype)
        result_row_list = parser.parse(file_stream)
        # Check all key are present in account.bank.statement.line !!
        if not result_row_list:
            raise osv.except_osv(_("Nothing to import"),
                                 _("The file is empty"))
        parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys()
        for col in parsed_cols:
            if col not in statement_line_obj._columns:
                raise osv.except_osv(_("Missing column !"),
                                     _("Column %s you try to import is not "
                                       "present in the bank statement line !") % col)

        statement_id = statement_obj.create(cr, uid,
                                            {'profile_id': prof.id},
                                            context=context)
        if prof.receivable_account_id:
            account_receivable = account_payable = prof.receivable_account_id.id
        else:
            account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(
                                                       cr, uid, context)
        try:
            # Record every line in the bank statement and compute the global commission
            # based on the commission_amount column
            statement_store = []
            for line in result_row_list:
                parser_vals = parser.get_st_line_vals(line)
                values = self.prepare_statetement_lines_vals(cr, uid, parser_vals, account_payable,
                                                             account_receivable, statement_id, context)
                statement_store.append(values)
            # Hack to bypass ORM poor perfomance. Sob...
            statement_line_obj._insert_lines(cr, uid, statement_store, context=context)

            # Build and create the global commission line for the whole statement
            comm_vals = self.prepare_global_commission_line_vals(cr, uid, parser, result_row_list,
                                                                 prof, statement_id, context)
            if comm_vals:
                statement_line_obj.create(cr, uid, comm_vals, context=context)
            else:
                # Trigger store field computation if someone has better idea
                start_bal = statement_obj.read(cr, uid, statement_id,
                                               ['balance_start'],
                                               context=context)
                start_bal = start_bal['balance_start']
                statement_obj.write(cr, uid, [statement_id],
                                    {'balance_start': start_bal})

            attachment_obj.create(cr,
                                  uid,
                                  {'name': 'statement file',
                                   'datas': file_stream,
                                   'datas_fname': "%s.%s" % (
                                       datetime.datetime.now().date(),
                                       ftype),
                                   'res_model': 'account.bank.statement',
                                   'res_id': statement_id},
                                  context=context)

            # If user ask to launch completion at end of import, do it !
            if prof.launch_import_completion:
                statement_obj.button_auto_completion(cr, uid, [statement_id], context)

            # Write the needed log infos on profile
            self.write_logs_after_import(cr, uid, prof.id,
                                         statement_id,
                                         len(result_row_list),
                                         context)

        except Exception:
            statement_obj.unlink(cr, uid, [statement_id], context=context)
            error_type, error_value, trbk = sys.exc_info()
            st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value)
            st += ''.join(traceback.format_tb(trbk, 30))
            raise osv.except_osv(_("Statement import error"),
                                 _("The statement cannot be created : %s") % st)
        return statement_id

class AccountStatementLine(Model):
    """
    Add sparse field on the statement line to allow to store all the
    bank infos that are given by an office. In this basic sample case
    it concern only commission_amount.
    """
    _inherit = "account.bank.statement.line"

    def _get_available_columns(self, statement_store):
        """Return writeable by SQL columns"""
        statement_line_obj = self.pool['account.bank.statement.line']
        model_cols = statement_line_obj._columns
        avail = [k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct')]
        keys = [k for k in statement_store[0].keys() if k in avail]
        keys.sort()
        return keys

    def _insert_lines(self, cr, uid, statement_store, context=None):
        """ Do raw insert into database because ORM is awfully slow
            when doing batch write. It is a shame that batch function
            does not exist"""
        statement_line_obj = self.pool['account.bank.statement.line']
        statement_line_obj.check_access_rule(cr, uid, [], 'create')
        statement_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
        cols = self._get_available_columns(statement_store)
        tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
        sql = "INSERT INTO account_bank_statement_line (%s) VALUES (%s);" % tmp_vals
        try:
            cr.executemany(sql, tuple(statement_store))
        except psycopg2.Error as sql_err:
            cr.rollback()
            raise osv.except_osv(_("ORM bypass error"),
                                 sql_err.pgerror)

    def _update_line(self, cr, uid, vals, context=None):
        """ Do raw update into database because ORM is awfully slow
            when doing batch write. It is a shame that batch function
            does not exist"""
        cols = self._get_available_columns([vals])
        tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols]))
        sql = "UPDATE account_bank_statement_line SET %s where id = %%(id)s;" % tmp_vals
        try:
            cr.execute(sql, vals)
        except psycopg2.Error as sql_err:
            cr.rollback()
            raise osv.except_osv(_("ORM bypass error"),
                                 sql_err.pgerror)

    _columns = {
        'commission_amount': fields.sparse(
            type='float',
            string='Line Commission Amount',
            serialization_field='additionnal_bank_fields'),
    }
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""Work with api.yaml files from a BMI description."""

import os

import yaml

from .utils import cd, check_output
from .errors import (ParseError, MissingKeyError, UnknownKeyError,
                     MissingFileError)
# from . import build
from .utils import read_first_of

_API_FILES = [os.path.join('.bmi', 'api.yaml'),
              os.path.join('.bmi', 'api.yml')]

_REQUIRED_KEYS = set(['language', ])
_OPTIONAL_KEYS = set(['name', 'type', 'register', 'class', 'package',
                      'libs', 'cflags', 'includes', 'build', 'path',
                      'initialize_args'])
_VALID_KEYS = _REQUIRED_KEYS | _OPTIONAL_KEYS

def validate_api(api):
    """Validate API description or raise an exception.

    Parameters
    ----------
    api : dict_like
        Description of a BMI.

    Raises
    ------
    ParseError
        If the description object is invalid.
    MissingKeyError
        If a required key is missing
    UnknownKeyError
        If an unknown key is found
    """
    if not isinstance(api, dict):
        raise ParseError('object is not a dictionary')

    found_keys = set(api.keys())

    if not found_keys.issuperset(_REQUIRED_KEYS):
        raise MissingKeyError(_REQUIRED_KEYS - found_keys)

    if not found_keys.issubset(_VALID_KEYS):
        raise UnknownKeyError(found_keys - _VALID_KEYS)

    return True

def prepend_env_path(path_var, path, env=None):
    """Prepend a path to and environment variable.

    Parameters
    ----------
    path_var : str
        Environment variable.
    path : str
        Path to prepend.
    env : dict, optional
        Environment to use, otherwise the current environment.
    """
    env = env or os.environ

    try:
        if path not in env[path_var]:
            env[path_var] = os.pathsep.join([path, env[path_var]])
    except KeyError:
        env[path_var] = path

def pkg_config(package, opt):
    """Use pkg-config to get compile options.

    Parameters
    ----------
    package : str
        Name of pkg-config package.
    opt : {'--cflags', '--libs'}
        Options for pkg-config.

    Returns
    -------
    str
        Compiler flags.
    """
    import copy

    assert(opt in ['--cflags', '--libs'])

    env = copy.copy(os.environ)
    if 'CSDMS_PREFIX' in env:
        pkg_config_path = os.path.join(env['CSDMS_PREFIX'], 'lib', 'pkgconfig')
        prepend_env_path('PKG_CONFIG_PATH', pkg_config_path, env=env)

    return check_output(['pkg-config', opt, package], env=env).strip()

def find_api_description_file(dir='.'):
    with cd(dir):
        (_, contents) = read_first_of(['.bmi.yaml', '.bmi.yml'])
        desc = yaml.load(contents)
        if 'implementation' in desc:
            if isinstance(desc['implementation'], dict):
                return desc['implementation']

_POSSIBLE_API_NAMES = [
    os.path.join('.bmi', 'api.yaml'),
    os.path.join('.bmi', 'api.yml'),
    'api.yaml',
    'api.yml',
]

def load_api(fname):
    with cd(os.path.dirname(fname)):
        (path_to_api, contents) = read_first_of(_POSSIBLE_API_NAMES)
        api = yaml.load(contents)
        api['path'] = os.path.dirname(path_to_api)

    validate_api(api)

    return api

def load(dir='.'):
    """Load an api description from a file.

    Parameters
    ----------
    dir : str, optional
        Path to folder that contains description file.

    Returns
    -------
    dict
        Description of the API.

    Raises
    ------
    ParseError
    MissingKeyError
    UnknownKeyError
        If the description is invalid.
    MissingFileError
        If an api file cannot be read.
    """
    with cd(dir):
        (path_to_api, contents) = read_first_of(
            [os.path.join('.bmi', 'api.yaml'),
             os.path.join('.bmi', 'api.yml'), 'api.yaml', 'api.yml'])
        api = yaml.load(contents)
        api['path'] = os.path.dirname(path_to_api)

    validate_api(api)

    api.pop('build', None)

    if api['language'] == 'c':
        cflags = api['cflags']
        if isinstance(cflags, dict) and 'pkgconfig' in cflags:
            api['cflags'] = pkg_config(cflags['pkgconfig'], '--cflags')

        libs = api['libs']
        if isinstance(libs, dict) and 'pkgconfig' in libs:
            api['libs'] = pkg_config(cflags['pkgconfig'], '--libs')
    elif api['language'] == 'python':
        pass

    return api

def load_all(dir='.'):
    """Load multiple API descriptions.

    Parameters
    ----------
    dir : str, optional
        Path to folder that contains description file.

    Returns
    -------
    list
        Descriptions of APIs.
    """
    with cd(dir):
        try:
            (_, contents) = read_first_of(['.bmi.yaml', '.bmi.yml'])
        except MissingFileError:
            apis = [load()]
        else:
            bmi = yaml.load(contents)
            apis = [load(dir=path) for path in bmi['bmi_paths']]

    return apis

def get_subdirs(dir='.'):
    return next(os.walk(os.path.join(dir)))[1]

def is_bmi_metadata_folder(path):
    try:
        with open(os.path.join(path, 'api.yaml'), 'r') as fp:
            yaml.load(fp)
    except IOError, yaml.error.YAMLError:
        return False
    else:
        return True

def find_bmi_metadata(dir='.'):
    """Find BMI metadata in a directory tree.

    A folder is thought to contain BMI metadata if:

    *  Its name is `.bmi` and contains a `api.yaml`. If it contains only
       folders, each folder is assumed to contain bmi metadata for an
       individual model.
    *  It is listed in `.bmi.yaml` file under `bmi_paths`. If any of these
       paths do not end in `.bmi`, but there is a `.bmi` folder there, that
       folder is used.

    Parameters
    ----------
    dir : str, optional
        Path to folder to search for metadata.

    Returns
    -------
    tuple of str
        Paths to BMI metadata folders.
    """
    found = set()

    if not os.path.isdir(dir):
        raise ValueError('{dir}: not a directory'.format(dir=dir))

    for root, dirs, files in os.walk(dir):
        paths = []

        if '.bmi' in dirs:
            paths.append('.bmi')
            paths.extend(get_subdirs(os.path.join(root, '.bmi')))

        for fname in ('.bmi.yaml', '.bmi.yml'):
            try:
                bmi_paths = load_bmi_paths(fname, dir=root)
            except IOError:
                pass
            except yaml.error.YAMLError:
                pass
            else:
                paths.extend(bmi_paths)

        for path in paths:
            found.add(os.path.normpath(os.path.join(root, path)))

    for path in tuple(found):
        if not is_bmi_metadata_folder(path):
            found.remove(path)

    return tuple(found)

def load_bmi_paths(fname, dir='.'):
    paths = []
    with cd(dir):
        with open(fname, 'r') as fp:
            bmi = yaml.load(fp)

        for bmi_path in bmi.get('bmi_paths', []):
            if os.path.isdir(bmi_path):
                if os.path.isdir(os.path.join(bmi_path, '.bmi')):
                    bmi_path = os.path.join(bmi_path, '.bmi')
                paths.append(bmi_path)

    return paths
<commit_msg>Get directory only if path is to a file.<commit_after>#! /usr/bin/env python
"""Work with api.yaml files from a BMI description."""

import os

import yaml

from .utils import cd, check_output
from .errors import (ParseError, MissingKeyError, UnknownKeyError,
                     MissingFileError)
# from . import build
from .utils import read_first_of

_API_FILES = [os.path.join('.bmi', 'api.yaml'),
              os.path.join('.bmi', 'api.yml')]

_REQUIRED_KEYS = set(['language', ])
_OPTIONAL_KEYS = set(['name', 'type', 'register', 'class', 'package',
                      'libs', 'cflags', 'includes', 'build', 'path',
                      'initialize_args'])
_VALID_KEYS = _REQUIRED_KEYS | _OPTIONAL_KEYS

def validate_api(api):
    """Validate API description or raise an exception.

    Parameters
    ----------
    api : dict_like
        Description of a BMI.

    Raises
    ------
    ParseError
        If the description object is invalid.
    MissingKeyError
        If a required key is missing
    UnknownKeyError
        If an unknown key is found
    """
    if not isinstance(api, dict):
        raise ParseError('object is not a dictionary')

    found_keys = set(api.keys())

    if not found_keys.issuperset(_REQUIRED_KEYS):
        raise MissingKeyError(_REQUIRED_KEYS - found_keys)

    if not found_keys.issubset(_VALID_KEYS):
        raise UnknownKeyError(found_keys - _VALID_KEYS)

    return True

def prepend_env_path(path_var, path, env=None):
    """Prepend a path to and environment variable.

    Parameters
    ----------
    path_var : str
        Environment variable.
    path : str
        Path to prepend.
    env : dict, optional
        Environment to use, otherwise the current environment.
    """
    env = env or os.environ

    try:
        if path not in env[path_var]:
            env[path_var] = os.pathsep.join([path, env[path_var]])
    except KeyError:
        env[path_var] = path

def pkg_config(package, opt):
    """Use pkg-config to get compile options.

    Parameters
    ----------
    package : str
        Name of pkg-config package.
    opt : {'--cflags', '--libs'}
        Options for pkg-config.

    Returns
    -------
    str
        Compiler flags.
    """
    import copy

    assert(opt in ['--cflags', '--libs'])

    env = copy.copy(os.environ)
    if 'CSDMS_PREFIX' in env:
        pkg_config_path = os.path.join(env['CSDMS_PREFIX'], 'lib', 'pkgconfig')
        prepend_env_path('PKG_CONFIG_PATH', pkg_config_path, env=env)

    return check_output(['pkg-config', opt, package], env=env).strip()

def find_api_description_file(dir='.'):
    with cd(dir):
        (_, contents) = read_first_of(['.bmi.yaml', '.bmi.yml'])
        desc = yaml.load(contents)
        if 'implementation' in desc:
            if isinstance(desc['implementation'], dict):
                return desc['implementation']

_POSSIBLE_API_NAMES = [
    os.path.join('.bmi', 'api.yaml'),
    os.path.join('.bmi', 'api.yml'),
    'api.yaml',
    'api.yml',
]

def load_api(fname):
    if os.path.isfile(fname):
        path_to_bmi = os.path.dirname(fname)
    else:
        path_to_bmi = fname

    with cd(path_to_bmi):
        (path_to_api, contents) = read_first_of(_POSSIBLE_API_NAMES)
        api = yaml.load(contents)
        api['path'] = os.path.dirname(path_to_api)

    validate_api(api)

    return api

def load(dir='.'):
    """Load an api description from a file.

    Parameters
    ----------
    dir : str, optional
        Path to folder that contains description file.

    Returns
    -------
    dict
        Description of the API.

    Raises
    ------
    ParseError
    MissingKeyError
    UnknownKeyError
        If the description is invalid.
    MissingFileError
        If an api file cannot be read.
    """
    with cd(dir):
        (path_to_api, contents) = read_first_of(
            [os.path.join('.bmi', 'api.yaml'),
             os.path.join('.bmi', 'api.yml'), 'api.yaml', 'api.yml'])
        api = yaml.load(contents)
        api['path'] = os.path.dirname(path_to_api)

    validate_api(api)

    api.pop('build', None)

    if api['language'] == 'c':
        cflags = api['cflags']
        if isinstance(cflags, dict) and 'pkgconfig' in cflags:
            api['cflags'] = pkg_config(cflags['pkgconfig'], '--cflags')

        libs = api['libs']
        if isinstance(libs, dict) and 'pkgconfig' in libs:
            api['libs'] = pkg_config(cflags['pkgconfig'], '--libs')
    elif api['language'] == 'python':
        pass

    return api

def load_all(dir='.'):
    """Load multiple API descriptions.

    Parameters
    ----------
    dir : str, optional
        Path to folder that contains description file.

    Returns
    -------
    list
        Descriptions of APIs.
    """
    with cd(dir):
        try:
            (_, contents) = read_first_of(['.bmi.yaml', '.bmi.yml'])
        except MissingFileError:
            apis = [load()]
        else:
            bmi = yaml.load(contents)
            apis = [load(dir=path) for path in bmi['bmi_paths']]

    return apis

def get_subdirs(dir='.'):
    return next(os.walk(os.path.join(dir)))[1]

def is_bmi_metadata_folder(path):
    try:
        with open(os.path.join(path, 'api.yaml'), 'r') as fp:
            yaml.load(fp)
    except IOError, yaml.error.YAMLError:
        return False
    else:
        return True

def find_bmi_metadata(dir='.'):
    """Find BMI metadata in a directory tree.

    A folder is thought to contain BMI metadata if:

    *  Its name is `.bmi` and contains a `api.yaml`. If it contains only
       folders, each folder is assumed to contain bmi metadata for an
       individual model.
    *  It is listed in `.bmi.yaml` file under `bmi_paths`. If any of these
       paths do not end in `.bmi`, but there is a `.bmi` folder there, that
       folder is used.

    Parameters
    ----------
    dir : str, optional
        Path to folder to search for metadata.

    Returns
    -------
    tuple of str
        Paths to BMI metadata folders.
    """
    found = set()

    if not os.path.isdir(dir):
        raise ValueError('{dir}: not a directory'.format(dir=dir))

    for root, dirs, files in os.walk(dir):
        paths = []

        if '.bmi' in dirs:
            paths.append('.bmi')
            paths.extend(get_subdirs(os.path.join(root, '.bmi')))

        for fname in ('.bmi.yaml', '.bmi.yml'):
            try:
                bmi_paths = load_bmi_paths(fname, dir=root)
            except IOError:
                pass
            except yaml.error.YAMLError:
                pass
            else:
                paths.extend(bmi_paths)

        for path in paths:
            found.add(os.path.normpath(os.path.join(root, path)))

    for path in tuple(found):
        if not is_bmi_metadata_folder(path):
            found.remove(path)

    return tuple(found)

def load_bmi_paths(fname, dir='.'):
    paths = []
    with cd(dir):
        with open(fname, 'r') as fp:
            bmi = yaml.load(fp)

        for bmi_path in bmi.get('bmi_paths', []):
            if os.path.isdir(bmi_path):
                if os.path.isdir(os.path.join(bmi_path, '.bmi')):
                    bmi_path = os.path.join(bmi_path, '.bmi')
                paths.append(bmi_path)

    return paths
<|endoftext|>