<commit_before>from django import forms
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.forms import FloatField
from django.utils.translation import gettext_lazy as _
from django.forms.models import inlineformset_factory

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Fieldset, Layout, Div, HTML

from geotrek.common.forms import CommonForm
from geotrek.core.fields import TopologyField
from geotrek.core.models import Topology

from .models import Intervention, InterventionJob, Project

class ManDayForm(forms.ModelForm):

    class Meta:
        fields = ('id', 'nb_days', 'job')

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.helper.layout = Layout('id', 'nb_days', 'job')
        self.fields['nb_days'].widget.attrs['placeholder'] = _('Days')
        self.fields['nb_days'].label = ''
        self.fields['nb_days'].widget.attrs['class'] = 'input-mini'
        self.fields['job'].widget.attrs['class'] = 'input-medium'
        self.fields['job'].queryset = InterventionJob.objects.filter(active=True)

ManDayFormSet = inlineformset_factory(Intervention, Intervention.jobs.through, form=ManDayForm, extra=1)

class FundingForm(forms.ModelForm):

    class Meta:
        fields = ('id', 'amount', 'organism')

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.helper.layout = Layout('id', 'amount', 'organism')
        self.fields['organism'].widget.attrs['class'] = 'input-xlarge'

FundingFormSet = inlineformset_factory(Project, Project.founders.through, form=FundingForm, extra=1)

class InterventionForm(CommonForm):
    """ An intervention can be a Point or a Line """

    topology = TopologyField(label="")
    length = FloatField(required=False, label=_("Length"))
    project = forms.ModelChoiceField(required=False, label=_("Project"),
                                     queryset=Project.objects.existing())

    geomfields = ['topology']
    leftpanel_scrollable = False

    fieldslayout = [
        Div(
            HTML(
                """

         {0}

         {1}

""".format(_("Main"), _("Advanced"))),
            Div(
                Div(
                    'structure',
                    'name',
                    'date',
                    'status',
                    'disorders',
                    'type',
                    'subcontracting',
                    'length',
                    'width',
                    'height',
                    'stake',
                    'project',
                    'description',
                    css_id="main",
                    css_class="tab-pane active"
                ),
                Div(
                    'material_cost',
                    'heliport_cost',
                    'subcontract_cost',
                    Fieldset(_("Mandays")),
                    css_id="advanced",  # used in Javascript for activating tab if error
                    css_class="tab-pane"
                ),
                css_class="scrollable tab-content"
            ),
            css_class="tabbable"
        ),
    ]

    class Meta(CommonForm.Meta):
        model = Intervention
        fields = CommonForm.Meta.fields + \
            ['structure', 'name', 'date', 'status', 'disorders', 'type', 'description', 'subcontracting', 'length', 'width',
             'height', 'stake', 'project', 'material_cost', 'heliport_cost', 'subcontract_cost', 'topology']

    def __init__(self, *args, target_type=None, target_id=None, **kwargs):
        super().__init__(*args, **kwargs)

        if not self.instance.pk:
            # New intervention. We have to set its target.
            if target_type and target_id:
                # Point target to an existing topology
                ct = ContentType.objects.get_for_id(target_type)
                self.instance.target = ct.get_object_for_this_type(id=target_id)
                # Set POST URL
                self.helper.form_action += '?target_type={}&target_id={}'.format(target_type, target_id)
            else:
                # Point target to a new topology
                self.instance.target = Topology(kind='INTERVENTION')
        # Else: existing intervention. Target is already set

        self.fields['topology'].initial = self.instance.target

        if self.instance.target.__class__ == Topology:
            # Intervention has its own topology
            title = _("On {}".format(_("Paths")))
            self.fields['topology'].label = \
                '{title}'.format(
                    prefix=settings.STATIC_URL, title=title
            )
        else:
            # Intervention on an existing topology
            icon = self.instance.target._meta.model_name
            title = _("On {}".format(str(self.instance.target)))
            self.fields['topology'].label = \
                '{title}'.format(
                    prefix=settings.STATIC_URL, icon=icon, title=title,
                    url=self.instance.target.get_detail_url()
            )
            # Topology is readonly
            del self.fields['topology']

        # Length is not editable in AltimetryMixin
        self.fields['length'].initial = self.instance.length
        editable = bool(self.instance.geom and (self.instance.geom.geom_type == 'Point'
                        or self.instance.geom.geom_type == 'LineString'))
        self.fields['length'].widget.attrs['readonly'] = editable

    def save(self, *args, **kwargs):
        target = self.instance.target
        if not target.pk:
            target.save()
        topology = self.cleaned_data.get('topology')
        if topology and topology.pk != target.pk:
            target.mutate(topology)
        intervention = super().save(*args, **kwargs, commit=False)
        intervention.target = target
        intervention.save()
        self.save_m2m()
        return intervention

class ProjectForm(CommonForm):
    fieldslayout = [
        Div(
            Div(
                Div('structure',
                    'name',
                    'type',
                    'domain',
                    'begin_year',
                    'end_year',
                    'constraint',
                    'global_cost',
                    'comments',

                    css_class="span6"),
                Div('project_owner',
                    'project_manager',
                    'contractors',
                    Fieldset(_("Fundings")),
                    css_class="span6"),
                css_class="row-fluid"
            ),
            css_class="container-fluid"
        ),
    ]

    class Meta(CommonForm.Meta):
        model = Project
        fields = CommonForm.Meta.fields + \
            ['structure', 'name', 'type', 'domain', 'begin_year', 'end_year', 'constraint',
             'global_cost', 'comments', 'project_owner', 'project_manager', 'contractors']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper.form_tag = False
<commit_msg>Enable original job selection in ManDay form even if job was deactivated<commit_after>from django import forms
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.forms import FloatField
from django.utils.translation import gettext_lazy as _
from django.forms.models import inlineformset_factory

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Fieldset, Layout, Div, HTML

from geotrek.common.forms import CommonForm
from geotrek.core.fields import TopologyField
from geotrek.core.models import Topology

from .models import Intervention, InterventionJob, Project

class ManDayForm(forms.ModelForm):

    class Meta:
        fields = ('id', 'nb_days', 'job')

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.helper.layout = Layout('id', 'nb_days', 'job')
        self.fields['nb_days'].widget.attrs['placeholder'] = _('Days')
        self.fields['nb_days'].label = ''
        self.fields['nb_days'].widget.attrs['class'] = 'input-mini'
        self.fields['job'].widget.attrs['class'] = 'input-medium'
        if self.instance and self.instance.pk:
            self.fields['job'].queryset = InterventionJob.objects.filter(Q(active=True) | Q(id=self.instance.job.id))
        else:
            self.fields['job'].queryset = InterventionJob.objects.filter(active=True)

ManDayFormSet = inlineformset_factory(Intervention, Intervention.jobs.through, form=ManDayForm, extra=1)

class FundingForm(forms.ModelForm):

    class Meta:
        fields = ('id', 'amount', 'organism')

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.helper.layout = Layout('id', 'amount', 'organism')
        self.fields['organism'].widget.attrs['class'] = 'input-xlarge'

FundingFormSet = inlineformset_factory(Project, Project.founders.through, form=FundingForm, extra=1)

class InterventionForm(CommonForm):
    """ An intervention can be a Point or a Line """

    topology = TopologyField(label="")
    length = FloatField(required=False, label=_("Length"))
    project = forms.ModelChoiceField(required=False, label=_("Project"),
                                     queryset=Project.objects.existing())

    geomfields = ['topology']
    leftpanel_scrollable = False

    fieldslayout = [
        Div(
            HTML(
                """

         {0}

         {1}

""".format(_("Main"), _("Advanced"))),
            Div(
                Div(
                    'structure',
                    'name',
                    'date',
                    'status',
                    'disorders',
                    'type',
                    'subcontracting',
                    'length',
                    'width',
                    'height',
                    'stake',
                    'project',
                    'description',
                    css_id="main",
                    css_class="tab-pane active"
                ),
                Div(
                    'material_cost',
                    'heliport_cost',
                    'subcontract_cost',
                    Fieldset(_("Mandays")),
                    css_id="advanced",  # used in Javascript for activating tab if error
                    css_class="tab-pane"
                ),
                css_class="scrollable tab-content"
            ),
            css_class="tabbable"
        ),
    ]

    class Meta(CommonForm.Meta):
        model = Intervention
        fields = CommonForm.Meta.fields + \
            ['structure', 'name', 'date', 'status', 'disorders', 'type', 'description', 'subcontracting', 'length', 'width',
             'height', 'stake', 'project', 'material_cost', 'heliport_cost', 'subcontract_cost', 'topology']

    def __init__(self, *args, target_type=None, target_id=None, **kwargs):
        super().__init__(*args, **kwargs)

        if not self.instance.pk:
            # New intervention. We have to set its target.
            if target_type and target_id:
                # Point target to an existing topology
                ct = ContentType.objects.get_for_id(target_type)
                self.instance.target = ct.get_object_for_this_type(id=target_id)
                # Set POST URL
                self.helper.form_action += '?target_type={}&target_id={}'.format(target_type, target_id)
            else:
                # Point target to a new topology
                self.instance.target = Topology(kind='INTERVENTION')
        # Else: existing intervention. Target is already set

        self.fields['topology'].initial = self.instance.target

        if self.instance.target.__class__ == Topology:
            # Intervention has its own topology
            title = _("On {}".format(_("Paths")))
            self.fields['topology'].label = \
                '{title}'.format(
                    prefix=settings.STATIC_URL, title=title
            )
        else:
            # Intervention on an existing topology
            icon = self.instance.target._meta.model_name
            title = _("On {}".format(str(self.instance.target)))
            self.fields['topology'].label = \
                '{title}'.format(
                    prefix=settings.STATIC_URL, icon=icon, title=title,
                    url=self.instance.target.get_detail_url()
            )
            # Topology is readonly
            del self.fields['topology']

        # Length is not editable in AltimetryMixin
        self.fields['length'].initial = self.instance.length
        editable = bool(self.instance.geom and (self.instance.geom.geom_type == 'Point'
                        or self.instance.geom.geom_type == 'LineString'))
        self.fields['length'].widget.attrs['readonly'] = editable

    def save(self, *args, **kwargs):
        target = self.instance.target
        if not target.pk:
            target.save()
        topology = self.cleaned_data.get('topology')
        if topology and topology.pk != target.pk:
            target.mutate(topology)
        intervention = super().save(*args, **kwargs, commit=False)
        intervention.target = target
        intervention.save()
        self.save_m2m()
        return intervention

class ProjectForm(CommonForm):
    fieldslayout = [
        Div(
            Div(
                Div('structure',
                    'name',
                    'type',
                    'domain',
                    'begin_year',
                    'end_year',
                    'constraint',
                    'global_cost',
                    'comments',

                    css_class="span6"),
                Div('project_owner',
                    'project_manager',
                    'contractors',
                    Fieldset(_("Fundings")),
                    css_class="span6"),
                css_class="row-fluid"
            ),
            css_class="container-fluid"
        ),
    ]

    class Meta(CommonForm.Meta):
        model = Project
        fields = CommonForm.Meta.fields + \
            ['structure', 'name', 'type', 'domain', 'begin_year', 'end_year', 'constraint',
             'global_cost', 'comments', 'project_owner', 'project_manager', 'contractors']

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.helper.form_tag = False
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# localbuild.py - Clone a repository locally and run .gopherci/pipeline.sh

# localbuild is one of possibly several build types for a repository. When
# called with two arguments, they are interpreted as the repository and branch
# of a primary build. When called with four arguments, the first two are the
# repository and branch to build, and the second two are the repository and
# branch that triggered the build.
#
# The build type is responsible for calling Exclusive, setting up the build
# directory, and adding the initial pipeline tasks. All other
# pipeline/dependency logic is in gopherci.

import os
import re
import sys
sys.path.append("%s/lib" % os.getenv("GOPHER_INSTALLDIR"))
from gopherbot_v2 import Robot

bot = Robot()

# Pop off the executable path
sys.argv.pop(0)

# localbuild and all build types always get a "command" from gopherci,
# never triggered.
# One of "build", "depbuild", "job"
command = sys.argv.pop(0)

repository = sys.argv.pop(0)
branch = sys.argv.pop(0)

bot.SetParameter("GOPHERCI_BRANCH", branch)
pipeline = "pipeline"

if command == "job":
    pipeline = sys.argv.pop(0)
    bot.SetParameter("GOPHERCI_CUSTOM_PIPELINE", pipeline)

if command == "depbuild":
    deprepo = sys.argv.pop(0)
    depbranch = sys.argv.pop(0)
    bot.SetParameter("GOPHERCI_DEPBUILD", "true")
    bot.SetParameter("GOPHERCI_DEPREPO", deprepo)
    bot.SetParameter("GOPHERCI_DEPBRANCH", depbranch)

repodata = bot.GetRepoData()

if repository in repodata:
    repoconf = repodata[repository]

print("Repoconf:", repoconf)

if "CloneURL" not in repoconf:
    bot.Say("No 'clone_url' specified for '%s' in repositories.yaml" % repository)
    exit()
clone_url = repoconf["CloneURL"]

if "KeepHistory" not in repoconf:
    bot.Log("Debug", "DEBUG: defaulting keep_history to -1")
    keep_history = -1
else:
    keep_history = repoconf["KeepHistory"]

# Protect the repository directory with Exclusive
if not bot.Exclusive(repository, False):
    bot.Log("Warn", "Build of '%s' already in progress, exiting" % repository)
    if len(bot.user) > 0:
        bot.Say("build for '%s' already in progress, not starting a new build for branch '%s'" % (repository, branch))
    exit()

repobranch = "%s/%s" % (repository, branch)
bot.ExtendNamespace(repobranch, keep_history)

bot.AddTask("start-build", [])
bot.AddTask("git-init", [ clone_url ])
# Start with a clean jobdir
bot.AddTask("cleanup", [ repository ])
bot.AddTask("git-clone", [ clone_url, branch, repository, "true" ])
bot.AddTask("run-pipeline", [ pipeline ] + sys.argv)
bot.FinalTask("finish-build", [])
<commit_msg>Fixes for robot.Repository struct<commit_after>#!/usr/bin/env python3

# localbuild.py - Clone a repository locally and run .gopherci/pipeline.sh

# localbuild is one of possibly several build types for a repository. When
# called with two arguments, they are interpreted as the repository and branch
# of a primary build. When called with four arguments, the first two are the
# repository and branch to build, and the second two are the repository and
# branch that triggered the build.
#
# The build type is responsible for calling Exclusive, setting up the build
# directory, and adding the initial pipeline tasks. All other
# pipeline/dependency logic is in gopherci.

import os
import re
import sys
sys.path.append("%s/lib" % os.getenv("GOPHER_INSTALLDIR"))
from gopherbot_v2 import Robot

bot = Robot()

# Pop off the executable path
sys.argv.pop(0)

# localbuild and all build types always get a "command" from gopherci,
# never triggered.
# One of "build", "depbuild", "job"
command = sys.argv.pop(0)

repository = sys.argv.pop(0)
branch = sys.argv.pop(0)

bot.SetParameter("GOPHERCI_BRANCH", branch)
pipeline = "pipeline"

if command == "job":
    pipeline = sys.argv.pop(0)
    bot.SetParameter("GOPHERCI_CUSTOM_PIPELINE", pipeline)

if command == "depbuild":
    deprepo = sys.argv.pop(0)
    depbranch = sys.argv.pop(0)
    bot.SetParameter("GOPHERCI_DEPBUILD", "true")
    bot.SetParameter("GOPHERCI_DEPREPO", deprepo)
    bot.SetParameter("GOPHERCI_DEPBRANCH", depbranch)

repodata = bot.GetRepoData()

if repository in repodata:
    repoconf = repodata[repository]

clone_url = repoconf["CloneURL"]
if len(clone_url) == 0:
    bot.Say("No 'clone_url' specified for '%s' in repositories.yaml" % repository)
    exit(1)

keep_history = repoconf["KeepHistory"]
if keep_history == None:
    keep_history = -1

# Protect the repository directory with Exclusive
if not bot.Exclusive(repository, False):
    bot.Log("Warn", "Build of '%s' already in progress, exiting" % repository)
    if len(bot.user) > 0:
        bot.Say("build for '%s' already in progress, not starting a new build for branch '%s'" % (repository, branch))
    exit()

repobranch = "%s/%s" % (repository, branch)
bot.ExtendNamespace(repobranch, keep_history)

bot.AddTask("start-build", [])
bot.AddTask("git-init", [ clone_url ])
# Start with a clean jobdir
bot.AddTask("cleanup", [ repository ])
bot.AddTask("git-clone", [ clone_url, branch, repository, "true" ])
bot.AddTask("run-pipeline", [ pipeline ] + sys.argv)
bot.FinalTask("finish-build", [])
<|endoftext|>


<commit_before># Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from nose.tools import assert_equal
from botocore.session import get_session

SERVICE_RENAMES = {
    # Actual service name we use -> Allowed computed service name.
    'alexaforbusiness': 'alexa-for-business',
    'apigateway': 'api-gateway',
    'application-autoscaling': 'application-auto-scaling',
    'appmesh': 'app-mesh',
    'autoscaling': 'auto-scaling',
    'autoscaling-plans': 'auto-scaling-plans',
    'ce': 'cost-explorer',
    'cloudhsmv2': 'cloudhsm-v2',
    'cloudsearchdomain': 'cloudsearch-domain',
    'cognito-idp': 'cognito-identity-provider',
    'config': 'config-service',
    'cur': 'cost-and-usage-report-service',
    'datapipeline': 'data-pipeline',
    'directconnect': 'direct-connect',
    'devicefarm': 'device-farm',
    'discovery': 'application-discovery-service',
    'dms': 'database-migration-service',
    'ds': 'directory-service',
    'dynamodbstreams': 'dynamodb-streams',
    'elasticbeanstalk': 'elastic-beanstalk',
    'elastictranscoder': 'elastic-transcoder',
    'elb': 'elastic-load-balancing',
    'elbv2': 'elastic-load-balancing-v2',
    'es': 'elasticsearch-service',
    'events': 'cloudwatch-events',
    'globalaccelerator': 'global-accelerator',
    'iot-data': 'iot-data-plane',
    'iot-jobs-data': 'iot-jobs-data-plane',
    'iot1click-devices': 'iot-1click-devices-service',
    'iot1click-projects': 'iot-1click-projects',
    'kinesisanalytics': 'kinesis-analytics',
    'kinesisanalyticsv2': 'kinesis-analytics-v2',
    'kinesisvideo': 'kinesis-video',
    'lex-models': 'lex-model-building-service',
    'lex-runtime': 'lex-runtime-service',
    'logs': 'cloudwatch-logs',
    'machinelearning': 'machine-learning',
    'marketplacecommerceanalytics': 'marketplace-commerce-analytics',
    'marketplace-entitlement': 'marketplace-entitlement-service',
    'meteringmarketplace': 'marketplace-metering',
    'mgh': 'migration-hub',
    'sms-voice': 'pinpoint-sms-voice',
    'resourcegroupstaggingapi': 'resource-groups-tagging-api',
    'route53': 'route-53',
    'route53domains': 'route-53-domains',
    's3control': 's3-control',
    'sdb': 'simpledb',
    'secretsmanager': 'secrets-manager',
    'serverlessrepo': 'serverlessapplicationrepository',
    'servicecatalog': 'service-catalog',
    'stepfunctions': 'sfn',
    'storagegateway': 'storage-gateway',
}

ENDPOINT_PREFIX_OVERRIDE = {
    # entry in endpoints.json -> actual endpoint prefix.
    # The autoscaling-* services actually send requests to the
    # autoscaling service, but they're exposed as separate clients
    # in botocore.
    'autoscaling-plans': 'autoscaling',
    'application-autoscaling': 'autoscaling',
    # For neptune, we send requests to the RDS endpoint.
    'neptune': 'rds',
}
NOT_SUPPORTED_IN_SDK = [
    'mobileanalytics',
]

def test_endpoint_matches_service():
    # This verifies client names match up with data from the endpoints.json
    # file.  We want to verify that every entry in the endpoints.json
    # file corresponds to a client we can construct via
    # session.create_client(...).
    # So first we get a list of all the service names in the endpoints
    # file.
    session = get_session()
    loader = session.get_component('data_loader')
    endpoints = loader.load_data('endpoints')
    # A service can be in multiple partitions so we're using
    # a set here to remove dupes.
    services_in_endpoints_file = set([])
    for partition in endpoints['partitions']:
        for service in partition['services']:
            # There are some services we don't support in the SDK
            # so we don't need to add them to the list of services
            # we need to check.
            if service not in NOT_SUPPORTED_IN_SDK:
                services_in_endpoints_file.add(service)

    # Now we need to cross check them against services we know about.
    # The entries in endpoints.json are keyed off of the endpoint
    # prefix.  We don't directly have that data, so we have to load
    # every service model and look up its endpoint prefix in its
    # ``metadata`` section.
    known_services = loader.list_available_services('service-2')
    known_endpoint_prefixes = [
        session.get_service_model(service_name).endpoint_prefix
        for service_name in known_services
    ]

    # Now we go through every known endpoint prefix in the endpoints.json
    # file and ensure it maps to an endpoint prefix we've seen
    # in a service model.
    for endpoint_prefix in services_in_endpoints_file:
        # Check for an override where we know that an entry
        # in the endpoints.json actually maps to a different endpoint
        # prefix.
        endpoint_prefix = ENDPOINT_PREFIX_OVERRIDE.get(endpoint_prefix,
                                                       endpoint_prefix)
        yield (_assert_known_endpoint_prefix,
               endpoint_prefix,
               known_endpoint_prefixes)

def _assert_known_endpoint_prefix(endpoint_prefix, known_endpoint_prefixes):
    assert endpoint_prefix in known_endpoint_prefixes

def test_service_name_matches_endpoint_prefix():
    # Generates tests for each service to verify that the computed service
    # named based on the service id matches the service name used to
    # create a client (i.e the directory name in botocore/data)
    # unless there is an explicit exception.
    session = get_session()
    loader = session.get_component('data_loader')

    # Load the list of available services. The names here represent what
    # will become the client names.
    services = loader.list_available_services('service-2')

    for service in services:
        yield _assert_service_name_matches_endpoint_prefix, session, service

def _assert_service_name_matches_endpoint_prefix(session, service_name):
    service_model = session.get_service_model(service_name)
    computed_name = service_model.service_id.replace(' ', '-').lower()

    # Handle known exceptions where we have renamed the service directory
    # for one reason or another.
    actual_service_name = SERVICE_RENAMES.get(service_name, service_name)
    assert_equal(
        computed_name, actual_service_name,
        "Actual service name `%s` does not match expected service name "
        "we computed: `%s`" % (
            actual_service_name, computed_name))
<commit_msg>Add override for docdb to map to rds<commit_after># Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from nose.tools import assert_equal
from botocore.session import get_session

SERVICE_RENAMES = {
    # Actual service name we use -> Allowed computed service name.
    'alexaforbusiness': 'alexa-for-business',
    'apigateway': 'api-gateway',
    'application-autoscaling': 'application-auto-scaling',
    'appmesh': 'app-mesh',
    'autoscaling': 'auto-scaling',
    'autoscaling-plans': 'auto-scaling-plans',
    'ce': 'cost-explorer',
    'cloudhsmv2': 'cloudhsm-v2',
    'cloudsearchdomain': 'cloudsearch-domain',
    'cognito-idp': 'cognito-identity-provider',
    'config': 'config-service',
    'cur': 'cost-and-usage-report-service',
    'datapipeline': 'data-pipeline',
    'directconnect': 'direct-connect',
    'devicefarm': 'device-farm',
    'discovery': 'application-discovery-service',
    'dms': 'database-migration-service',
    'ds': 'directory-service',
    'dynamodbstreams': 'dynamodb-streams',
    'elasticbeanstalk': 'elastic-beanstalk',
    'elastictranscoder': 'elastic-transcoder',
    'elb': 'elastic-load-balancing',
    'elbv2': 'elastic-load-balancing-v2',
    'es': 'elasticsearch-service',
    'events': 'cloudwatch-events',
    'globalaccelerator': 'global-accelerator',
    'iot-data': 'iot-data-plane',
    'iot-jobs-data': 'iot-jobs-data-plane',
    'iot1click-devices': 'iot-1click-devices-service',
    'iot1click-projects': 'iot-1click-projects',
    'kinesisanalytics': 'kinesis-analytics',
    'kinesisanalyticsv2': 'kinesis-analytics-v2',
    'kinesisvideo': 'kinesis-video',
    'lex-models': 'lex-model-building-service',
    'lex-runtime': 'lex-runtime-service',
    'logs': 'cloudwatch-logs',
    'machinelearning': 'machine-learning',
    'marketplacecommerceanalytics': 'marketplace-commerce-analytics',
    'marketplace-entitlement': 'marketplace-entitlement-service',
    'meteringmarketplace': 'marketplace-metering',
    'mgh': 'migration-hub',
    'sms-voice': 'pinpoint-sms-voice',
    'resourcegroupstaggingapi': 'resource-groups-tagging-api',
    'route53': 'route-53',
    'route53domains': 'route-53-domains',
    's3control': 's3-control',
    'sdb': 'simpledb',
    'secretsmanager': 'secrets-manager',
    'serverlessrepo': 'serverlessapplicationrepository',
    'servicecatalog': 'service-catalog',
    'stepfunctions': 'sfn',
    'storagegateway': 'storage-gateway',
}

ENDPOINT_PREFIX_OVERRIDE = {
    # entry in endpoints.json -> actual endpoint prefix.
    # The autoscaling-* services actually send requests to the
    # autoscaling service, but they're exposed as separate clients
    # in botocore.
    'autoscaling-plans': 'autoscaling',
    'application-autoscaling': 'autoscaling',
    # For neptune, we send requests to the RDS endpoint.
    'neptune': 'rds',
    'docdb': 'rds',
}
NOT_SUPPORTED_IN_SDK = [
    'mobileanalytics',
]

def test_endpoint_matches_service():
    # This verifies client names match up with data from the endpoints.json
    # file.  We want to verify that every entry in the endpoints.json
    # file corresponds to a client we can construct via
    # session.create_client(...).
    # So first we get a list of all the service names in the endpoints
    # file.
    session = get_session()
    loader = session.get_component('data_loader')
    endpoints = loader.load_data('endpoints')
    # A service can be in multiple partitions so we're using
    # a set here to remove dupes.
    services_in_endpoints_file = set([])
    for partition in endpoints['partitions']:
        for service in partition['services']:
            # There are some services we don't support in the SDK
            # so we don't need to add them to the list of services
            # we need to check.
            if service not in NOT_SUPPORTED_IN_SDK:
                services_in_endpoints_file.add(service)

    # Now we need to cross check them against services we know about.
    # The entries in endpoints.json are keyed off of the endpoint
    # prefix.  We don't directly have that data, so we have to load
    # every service model and look up its endpoint prefix in its
    # ``metadata`` section.
    known_services = loader.list_available_services('service-2')
    known_endpoint_prefixes = [
        session.get_service_model(service_name).endpoint_prefix
        for service_name in known_services
    ]

    # Now we go through every known endpoint prefix in the endpoints.json
    # file and ensure it maps to an endpoint prefix we've seen
    # in a service model.
    for endpoint_prefix in services_in_endpoints_file:
        # Check for an override where we know that an entry
        # in the endpoints.json actually maps to a different endpoint
        # prefix.
        endpoint_prefix = ENDPOINT_PREFIX_OVERRIDE.get(endpoint_prefix,
                                                       endpoint_prefix)
        yield (_assert_known_endpoint_prefix,
               endpoint_prefix,
               known_endpoint_prefixes)

def _assert_known_endpoint_prefix(endpoint_prefix, known_endpoint_prefixes):
    assert endpoint_prefix in known_endpoint_prefixes

def test_service_name_matches_endpoint_prefix():
    # Generates tests for each service to verify that the computed service
    # named based on the service id matches the service name used to
    # create a client (i.e the directory name in botocore/data)
    # unless there is an explicit exception.
    session = get_session()
    loader = session.get_component('data_loader')

    # Load the list of available services. The names here represent what
    # will become the client names.
    services = loader.list_available_services('service-2')

    for service in services:
        yield _assert_service_name_matches_endpoint_prefix, session, service

def _assert_service_name_matches_endpoint_prefix(session, service_name):
    service_model = session.get_service_model(service_name)
    computed_name = service_model.service_id.replace(' ', '-').lower()

    # Handle known exceptions where we have renamed the service directory
    # for one reason or another.
    actual_service_name = SERVICE_RENAMES.get(service_name, service_name)
    assert_equal(
        computed_name, actual_service_name,
        "Actual service name `%s` does not match expected service name "
        "we computed: `%s`" % (
            actual_service_name, computed_name))
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt

from mpl_toolkits.axes_grid1 import make_axes_locatable
from mpl_toolkits.basemap import Basemap

def plotStdErrorsOnMap(lons, lats, ses, numPts=None, basemap=None, ax=None,
                        ptids=None):
    basemap = basemap or Basemap()
    if ax is None:
        fig, ax = plt.subplots(1,1, figsize=(15,10))

    if numPts is None:
        s = 50
    else:
        normPts = numPts/float(max(numPts))
        s = 500*normPts

    datamax = np.mean(np.abs(ses)) + np.std(np.abs(ses))
    datamag = np.floor(np.log10(datamax))
    vmax = (10**datamag)*np.maximum(
            np.floor(datamax/(10**datamag))*2, 1.)
    vmin=-vmax

    basemap.drawcoastlines(color=(1,1,1,1), ax=ax, zorder=0)
    xs, ys = basemap(lons, lats)
    p = basemap.scatter(xs, ys, c=ses, s=s,
                        vmin=vmin, vmax=vmax, cmap='RdYlBu_r',
                        ax=ax, edgecolor='None', alpha=0.5)
    ax.set_axis_bgcolor((0,0,0,0.2))
    divider = make_axes_locatable(ax)
    cax = divider.append_axes('right', size='2%', pad=0.05)
    plt.colorbar(p, cax=cax, label='Standard Error (m)')

    if ptids is not None:
        for x, y, ptid in zip(xs, ys, ptids):
            ax.text(x, y, ptid, va='center', ha='center', fontsize=10,
            transform=ax.transData)

    # If scaling point size by number of points, add a label.
    if numPts is not None:
        samplept = int(5*10**(np.floor(np.log10(max(numPts)))-1))
        smaplept = np.floor((max(numPts) + min(numPts))/2)
        basemap.scatter([0.05], [0.05], c='k', alpha=0.75,
                        s=500*samplept/float(max(numPts)),
                        vmin=vmin, vmax=vmax, ax=ax,
                        edgecolor='None', transform=ax.transAxes)
        ax.text(0.065, 0.05, '- {0:d} observations at site'.format(samplept),
            transform=ax.transAxes, va='center', fontsize=12)

    return ax

def plotLocTimeseries(data, calc, ax=None, title=False, nbr=False, **kwargs):
    if ax is None:
        fig, ax = plt.subplots(1,1)

    marker = kwargs.get('marker', '+')
    color = kwargs.get('color', 'k')
    alpha = kwargs.get('alpha', 1.0)
    ms = kwargs.get('ms', 15)
    ls = kwargs.get('ls', 'None')

    ax.plot(data.ts, data.ys, marker=marker, color=color, alpha=alpha,
            ms=ms, ls=ls)
    ax.plot(calc.ts, calc.ys)

    if title:
        ax.set_title(str(data))
    elif nbr:
        ax.set_title(data.recnbr)
    ax.invert_xaxis()
    return plt.gca()
<commit_msg>Enhanced options for dataPlot.<commit_after>import numpy as np
import matplotlib.pyplot as plt

from mpl_toolkits.axes_grid1 import make_axes_locatable
from mpl_toolkits.basemap import Basemap

def plotStdErrorsOnMap(lons, lats, ses, numPts=None, basemap=None, ax=None,
                        ptids=None, clabel=None, vmax=None):
    basemap = basemap or Basemap()
    if ax is None:
        fig, ax = plt.subplots(1,1, figsize=(15,10))

    if numPts is None:
        s = 50
    else:
        normPts = numPts/float(max(numPts))
        s = 500*normPts

    clabel = clabel or 'Standard Error (m)'

    datamax = np.mean(np.abs(ses)) + np.std(np.abs(ses))
    datamag = np.floor(np.log10(datamax))
    if vmax is None:
        vmax = (10**datamag)*np.maximum(
                np.floor(datamax/(10**datamag))*2, 1.)

    if np.alltrue(ses >= 0):
        cmap = 'Reds'
        vmin = 0
    else:
        cmap = 'RdYlBu_r'
        vmin=-vmax

    basemap.drawcoastlines(color=(1,1,1,1), ax=ax, zorder=0)
    xs, ys = basemap(lons, lats)
    p = basemap.scatter(xs, ys, c=ses, s=s,
                        vmin=vmin, vmax=vmax, cmap=cmap,
                        ax=ax, edgecolor='None', alpha=0.5)
    ax.set_axis_bgcolor((0,0,0,0.2))
    divider = make_axes_locatable(ax)
    cax = divider.append_axes('right', size='2%', pad=0.05)
    plt.colorbar(p, cax=cax, label=clabel)

    if ptids is not None:
        for x, y, ptid in zip(xs, ys, ptids):
            ax.text(x, y, ptid, va='center', ha='center', fontsize=10,
            transform=ax.transData)

    # If scaling point size by number of points, add a label.
    if numPts is not None:
        samplept = int(5*10**(np.floor(np.log10(max(numPts)))-1))
        smaplept = np.floor((max(numPts) + min(numPts))/2)
        basemap.scatter([0.05], [0.05], c='k', alpha=0.75,
                        s=500*samplept/float(max(numPts)),
                        vmin=vmin, vmax=vmax, ax=ax,
                        edgecolor='None', transform=ax.transAxes)
        ax.text(0.065, 0.05, '- {0:d} observations at site'.format(samplept),
            transform=ax.transAxes, va='center', fontsize=12)

    return ax

def plotLocTimeseries(data, calc, ax=None, title=False, nbr=False, **kwargs):
    if ax is None:
        fig, ax = plt.subplots(1,1)

    marker = kwargs.get('marker', '+')
    color = kwargs.get('color', 'k')
    alpha = kwargs.get('alpha', 1.0)
    ms = kwargs.get('ms', 15)
    ls = kwargs.get('ls', 'None')

    ax.plot(data.ts, data.ys, marker=marker, color=color, alpha=alpha,
            ms=ms, ls=ls)
    ax.plot(calc.ts, calc.ys)

    if title:
        ax.set_title(str(data))
    elif nbr:
        ax.set_title(data.recnbr)
    ax.invert_xaxis()
    return plt.gca()
<|endoftext|>


<commit_before>"""
Debug cog.
"""
import inspect
import pprint
import traceback
import asyncio

import sys

import rethinkdb as r

import discord
from discord.ext import commands

from joku.bot import Jokusoramame, Context
from joku.checks import is_owner
from joku.cogs._common import Cog

class Debug(Cog):
    @commands.command(pass_context=True)
    @commands.check(is_owner)
    async def load(self, ctx, *, cog):
        ctx.bot.load_extension(cog)
        await ctx.bot.say(":heavy_check_mark: Loaded extension.")

    @commands.command(pass_context=True)
    @commands.check(is_owner)
    async def eval(self, ctx, *, cmd):
        try:
            d = eval(cmd, {
                "r": r, "asyncio": asyncio,
                "member": ctx.message.author, "message": ctx.message,
                "server": ctx.message.server, "channel": ctx.message.channel,
                "bot": ctx.bot, "self": self, "ctx": ctx,
                **sys.modules
                })
            if inspect.isawaitable(d):
                d = await d
        except Exception:
            f = "```{}```".format(''.join(traceback.format_exc()))
            await ctx.bot.say(f)
            return

        await ctx.bot.say("`" + repr(d) + "`")

    @commands.group()
    @commands.check(is_owner)
    async def debug(self):
        """
        Debug command to inspect the bot.

        Only usable by the owner.
        """

    @debug.command(pass_context=True)
    async def reload(self, ctx, module: str):
        try:
            ctx.bot.unload_extension(module)
            ctx.bot.load_extension(module)
        except Exception as e:
            await ctx.bot.say(e)
        else:
            await ctx.bot.say("Reloaded `{}`.".format(module))

    @debug.command(pass_context=True)
    async def punish(self, ctx: Context, user: discord.User):
        """
        Punishes a user.

        Sets their EXP to a very large negative number.
        """
        await ctx.bot.rethinkdb.update_user_xp(user, xp=-3.4756738956329854e+307)
        await ctx.bot.say(":skull: User {} has been punished.".format(user))

    @debug.command(pass_context=True)
    async def resetxp(self, ctx: Context, *, user: discord.User):
        """
        Resets a user's EXP to 0.
        """
        user = await ctx.bot.rethinkdb.get_user_xp(user)

        to_add = 0 - user
        await ctx.bot.rethinkdb.update_user_xp(user, xp=to_add)
        await ctx.bot.say(":put_litter_in_its_place: User {} has had their XP set to 0.".format(user))

    @debug.group()
    async def rdb(self):
        """
        Command group to inspect the RethinkDB status.
        """

    @rdb.command(pass_context=True)
    async def inspect(self, ctx, *, user: discord.Member):
        obb = await ctx.bot.rethinkdb.create_or_get_user(user)

        p = pprint.pformat(obb)
        await ctx.bot.say("```json\n{}\n```".format(p))

    @rdb.command(pass_context=True)
    async def info(self, ctx):
        """
        Gets data about the RethinkDB cluster.
        """
        data = await ctx.bot.rethinkdb.get_info()

        tmp = {
            "server": data["server_info"]["name"],
            "server_id": data["server_info"]["id"],
            "jobs": len(data["jobs"]),
            "clients": data["stats"]["query_engine"]["client_connections"]
        }

        await ctx.bot.say("""**RethinkDB stats**:

**Connected to server** `{server}` (`{server_id}`).
There are `{jobs}` job(s) across `{clients}` clients.
        """.format(**tmp))

def setup(bot):
    bot.add_cog(Debug(bot))
<commit_msg>Add update debug command.<commit_after>"""
Debug cog.
"""
import inspect
import pprint
import traceback
import asyncio

import sys

import rethinkdb as r

import discord
from discord.ext import commands

from joku.bot import Jokusoramame, Context
from joku.checks import is_owner
from joku.cogs._common import Cog

class Debug(Cog):
    @commands.command(pass_context=True)
    @commands.check(is_owner)
    async def load(self, ctx, *, cog):
        ctx.bot.load_extension(cog)
        await ctx.bot.say(":heavy_check_mark: Loaded extension.")

    @commands.command(pass_context=True)
    @commands.check(is_owner)
    async def eval(self, ctx, *, cmd):
        try:
            d = eval(cmd, {
                "r": r, "asyncio": asyncio,
                "member": ctx.message.author, "message": ctx.message,
                "server": ctx.message.server, "channel": ctx.message.channel,
                "bot": ctx.bot, "self": self, "ctx": ctx,
                **sys.modules
                })
            if inspect.isawaitable(d):
                d = await d
        except Exception:
            f = "```{}```".format(''.join(traceback.format_exc()))
            await ctx.bot.say(f)
            return

        await ctx.bot.say("`" + repr(d) + "`")

    @commands.group()
    @commands.check(is_owner)
    async def debug(self):
        """
        Debug command to inspect the bot.

        Only usable by the owner.
        """

    @debug.command(pass_context=True)
    async def update(self, ctx: Context):
        await ctx.bot.say("Pulling from Git...")

        process = await asyncio.create_subprocess_exec("git", "pull", stdout=asyncio.subprocess.PIPE,
                                                       stderr=asyncio.subprocess.PIPE)
        (stdout, stderr) = await process.communicate()

        if stdout:
            sem = discord.Embed(title="stdout", description="```\n" + stdout.decode() + "\n```")
            await ctx.bot.say(embed=sem)

        if stderr:
            rem = discord.Embed(description="```\n" + stderr.decode() + "\n```")
            await ctx.bot.say(embed=rem)

    @debug.command(pass_context=True)
    async def reload(self, ctx, module: str):
        try:
            ctx.bot.unload_extension(module)
            ctx.bot.load_extension(module)
        except Exception as e:
            await ctx.bot.say(e)
        else:
            await ctx.bot.say("Reloaded `{}`.".format(module))

    @debug.command(pass_context=True)
    async def punish(self, ctx: Context, user: discord.User):
        """
        Punishes a user.

        Sets their EXP to a very large negative number.
        """
        await ctx.bot.rethinkdb.update_user_xp(user, xp=-3.4756738956329854e+307)
        await ctx.bot.say(":skull: User {} has been punished.".format(user))

    @debug.command(pass_context=True)
    async def resetxp(self, ctx: Context, *, user: discord.User):
        """
        Resets a user's EXP to 0.
        """
        user = await ctx.bot.rethinkdb.get_user_xp(user)

        to_add = 0 - user["xp"]
        await ctx.bot.rethinkdb.update_user_xp(user, xp=to_add)
        await ctx.bot.say(":put_litter_in_its_place: User {} has had their XP set to 0.".format(user))

    @debug.group()
    async def rdb(self):
        """
        Command group to inspect the RethinkDB status.
        """

    @rdb.command(pass_context=True)
    async def inspect(self, ctx, *, user: discord.Member):
        obb = await ctx.bot.rethinkdb.create_or_get_user(user)

        p = pprint.pformat(obb)
        await ctx.bot.say("```json\n{}\n```".format(p))

    @rdb.command(pass_context=True)
    async def info(self, ctx):
        """
        Gets data about the RethinkDB cluster.
        """
        data = await ctx.bot.rethinkdb.get_info()

        tmp = {
            "server": data["server_info"]["name"],
            "server_id": data["server_info"]["id"],
            "jobs": len(data["jobs"]),
            "clients": data["stats"]["query_engine"]["client_connections"]
        }

        await ctx.bot.say("""**RethinkDB stats**:

**Connected to server** `{server}` (`{server_id}`).
There are `{jobs}` job(s) across `{clients}` clients.
        """.format(**tmp))

def setup(bot):
    bot.add_cog(Debug(bot))
<|endoftext|>


<commit_before>import feedparser
import requests

class JournalTOC(object):

    def getjournaltoc(self, keywords):
        results = []
        base = 'http://www.journaltocs.ac.uk/api/articles/'
        limit = 15
        params = {'to': limit}
        url = '%s%s' % (base, keywords)
        response = requests.get(url, params=params)
        print response.url
        rssdoc = feedparser.parse(response.content)
        for i in rssdoc.get('entries'):
            url = i.get('link')
            title = i.get('title')
            if str(i.get('date')) == 'None':
                dateall = ''
            else:
                dateall = str(i.get('date'))
            date = dateall[:10]
            result = {'type': 'journal-article', 'date': date, 'title': title, 'url': url, 'source': 'journaltoc'}
            if keywords.lower() not in result.get('title').lower():
                continue
            else:
                results.append(result)
        return results

#tests
test = 'Neural networks'
z = JournalTOC().getjournaltoc(test)
print z
<commit_msg>lower limit for jtoc<commit_after>import feedparser
import requests

class JournalTOC(object):

    def getjournaltoc(self, keywords):
        results = []
        base = 'http://www.journaltocs.ac.uk/api/articles/'
        limit = 10
        params = {'to': limit}
        url = '%s%s' % (base, keywords)
        response = requests.get(url, params=params)
        print response.url
        rssdoc = feedparser.parse(response.content)
        for i in rssdoc.get('entries'):
            url = i.get('link')
            title = i.get('title')
            if str(i.get('date')) == 'None':
                dateall = ''
            else:
                dateall = str(i.get('date'))
            date = dateall[:10]
            result = {'type': 'journal-article', 'date': date, 'title': title, 'url': url, 'source': 'journaltoc'}
            if keywords.lower() not in result.get('title').lower():
                continue
            else:
                results.append(result)
        return results

#tests
test = 'Neural networks'
z = JournalTOC().getjournaltoc(test)
print z
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import json
import math
import os
import shutil
import sys

from cctbx import sgtbx, crystal, uctbx
from cctbx.sgtbx import bravais_types

from xia2.Driver.DriverFactory import DriverFactory
from xia2.Handlers.Phil import PhilIndex
from xia2.Handlers.Streams import Chatter, Debug
from xia2.Handlers.Syminfo import Syminfo
# this was rather complicated - now simpler!
from xia2.lib.SymmetryLib import (clean_reindex_operator, lauegroup_to_lattice,
                                  spacegroup_name_xHM_to_old)
# XDS_ASCII meddling things
from xia2.Modules.XDS_ASCII import remove_misfits

def DialsSymmetry(DriverType = None):
  '''A factory for DialsSymmetryWrapper classes.'''

  DriverInstance = DriverFactory.Driver(DriverType)

  class DialsSymmetryWrapper(DriverInstance.__class__):
    '''A wrapper for dials.symmetry'''

    def __init__(self):
      # generic things
      super(DialsSymmetryWrapper, self).__init__()

      self.set_executable('dials.symmetry')

      self._input_laue_group = None

      self._experiments_filenames = []
      self._reflections_filenames = []
      self._output_experiments_filename = None
      self._output_reflections_filename = None

      self._hklin = None
      self._hklout = None
      self._pointgroup = None
      self._spacegroup = None
      self._reindex_matrix = None
      self._reindex_operator = None
      self._spacegroup_reindex_matrix = None
      self._spacegroup_reindex_operator = None
      self._confidence = 0.0
      self._hklref = None
      self._xdsin = None
      self._probably_twinned = False
      self._allow_out_of_sequence_files = False

      self._relative_length_tolerance = 0.05
      self._absolute_angle_tolerance = 2

      # space to store all possible solutions, to allow discussion of
      # the correct lattice with the indexer... this should be a
      # list containing e.g. 'tP'
      self._possible_lattices = []

      self._lattice_to_laue = { }

      # all "likely" spacegroups...
      self._likely_spacegroups = []

      # and unit cell information
      self._cell_info = { }
      self._cell = None

      self._json = None

    #def set_hklref(self, hklref):
      #self._hklref = hklref

    def set_hklin(self, hklin):
      self._hklin = hklin

    def get_hklin(self):
      return self._hklin

    def add_experiments(self, experiments):
      self._experiments_filenames.append(experiments)

    def add_reflections(self, reflections):
      self._reflections_filenames.append(reflections)

    def set_experiments_filename(self, experiments_filename):
      self._experiments_filenames = [experiments_filename]

    def set_reflections_filename(self, reflections_filename):
      self._reflections_filenames = [reflections_filename]

    def set_output_experiments_filename(self, experiments_filename):
      self._output_experiments_filename = experiments_filename

    def set_output_reflections_filename(self, reflections_filename):
      self._output_reflections_filename = reflections_filename

    def get_output_reflections_filename(self):
      return self._output_reflections_filename

    def get_output_experiments_filename(self):
      return self._output_experiments_filename

    def set_json(self, json):
      self._json = json

    def set_allow_out_of_sequence_files(self, allow=True):
      self._allow_out_of_sequence_files = allow

    def set_tolerance(self, relative_length_tolerance=0.05, absolute_angle_tolerance=2):
      self._relative_length_tolerance=relative_length_tolerance
      self._absolute_angle_tolerance=absolute_angle_tolerance

    #def get_hklref(self):
      #return self._hklref

    #def check_hklref(self):
      #if self._hklref is None:
        #raise RuntimeError('hklref not defined')
      #if not os.path.exists(self._hklref):
        #raise RuntimeError('hklref %s does not exist' % self._hklref)

    def set_correct_lattice(self, lattice):
      '''In a rerunning situation, set the correct lattice, which will
      assert a correct lauegroup based on the previous run of the
      program...'''

      if self._lattice_to_laue == { }:
        raise RuntimeError('no lattice to lauegroup mapping')

      if lattice not in self._lattice_to_laue:
        raise RuntimeError('lattice %s not possible' % lattice)

      self._input_laue_group = self._lattice_to_laue[lattice]

    def decide_pointgroup(self, ignore_errors=False, batches=None):
      '''Decide on the correct pointgroup for hklin.'''

      self.clear_command_line()

      if self._hklref:
        self.add_command_line('hklref')
        self.add_command_line(self._hklref)

      if self._hklin is not None:
        assert os.path.isfile(self._hklin)
        self.add_command_line("'%s'" % self._hklin)
      else:
        assert self._experiments_filenames# is not None
        assert self._reflections_filenames# is not None
        for exp in self._experiments_filenames:
          self.add_command_line("'%s'" % exp)
        for refl in self._reflections_filenames:
          self.add_command_line("'%s'" % refl)

        if not self._output_experiments_filename:
          self._output_experiments_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_experiments.json' % self.get_xpid())
        if not self._output_reflections_filename:
          self._output_reflections_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_reflections.pickle' % self.get_xpid())

        self.add_command_line("output.experiments='%s'" % self._output_experiments_filename)
        self.add_command_line("output.reflections='%s'" % self._output_reflections_filename)

      self.add_command_line('relative_length_tolerance=%s' % self._relative_length_tolerance)
      self.add_command_line('absolute_angle_tolerance=%s' % self._absolute_angle_tolerance)
      if not self._json:
        self._json = os.path.join(self.get_working_directory(),
                                '%d_dials_symmetry.json' % self.get_xpid())

      self.add_command_line("output.json='%s'" % self._json)

      self.start()

      self.close_wait()

      # check for errors
      self.check_for_errors()

      output = self.get_all_output()

      assert os.path.exists(self._json)
      with open(self._json, 'rb') as f:
        d = json.load(f)

      best_solution = d['subgroup_scores'][0]
      patterson_group = sgtbx.space_group(
        str(best_solution['patterson_group']))
      if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
        patterson_group = patterson_group.build_derived_acentric_group()

      cb_op_inp_min = sgtbx.change_of_basis_op(str(d['cb_op_inp_min']))
      cb_op_min_best = sgtbx.change_of_basis_op(str(best_solution['cb_op']))

      input_cell = uctbx.unit_cell(d['input_symmetry']['unit_cell'])
      cb_op_inp_best = cb_op_min_best * cb_op_inp_min
      best_cell = input_cell.change_basis(
        cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_best)))
      # ^^ not necessarily in reference setting e.g I2/m not C2/m

      cs = crystal.symmetry(unit_cell=best_cell,
        space_group=patterson_group)
      cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting()
      cs_reference = cs.as_reference_setting()
      self._pointgroup = cs_reference.space_group().type().lookup_symbol()

      self._confidence = best_solution['confidence']
      self._totalprob = best_solution['likelihood']
      cb_op_inp_best = cb_op_inp_best * cb_op_best_to_ref
      self._reindex_operator = cb_op_inp_best.as_xyz()
      self._reindex_matrix = cb_op_inp_best.c().r().as_double()

      if not self._input_laue_group and not self._hklref:
        for score in d['subgroup_scores']:
          patterson_group = sgtbx.space_group(str(score['patterson_group']))
          if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
            patterson_group = patterson_group.build_derived_acentric_group()

          cb_op_min_this = sgtbx.change_of_basis_op(str(score['cb_op']))
          cb_op_inp_this = cb_op_min_this * cb_op_inp_min
          unit_cell = input_cell.change_basis(
            cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_this)))
          cs = crystal.symmetry(unit_cell=unit_cell,
            space_group=patterson_group)
          cs_reference = cs.as_reference_setting()
          patterson_group = cs_reference.space_group()

          netzc = score['z_cc_net']
          # record this as a possible lattice if its Z score is positive
          lattice = str(bravais_types.bravais_lattice(
            group=patterson_group))
          if not lattice in self._possible_lattices:
            if netzc > 0.0:
              self._possible_lattices.append(lattice)
            self._lattice_to_laue[lattice] = patterson_group.type().lookup_symbol()
          self._likely_spacegroups.append(patterson_group.type().lookup_symbol())
      return

    def get_reindex_matrix(self):
      return self._reindex_matrix

    def get_reindex_operator(self):
      return self._reindex_operator

    def get_pointgroup(self):
      return self._pointgroup

    def get_cell(self):
      return self._cell

    def get_probably_twinned(self):
      return self._probably_twinned

    #def get_spacegroup(self):
      #return self._spacegroup

    #def get_spacegroup_reindex_operator(self):
      #return self._spacegroup_reindex_operator

    #def get_spacegroup_reindex_matrix(self):
      #return self._spacegroup_reindex_matrix

    # FIXME spacegroup != pointgroup
    decide_spacegroup = decide_pointgroup
    get_spacegroup = get_pointgroup
    get_spacegroup_reindex_operator = get_reindex_operator
    get_spacegroup_reindex_matrix = get_reindex_matrix

    def get_likely_spacegroups(self):
      return self._likely_spacegroups

    def get_confidence(self):
      return self._confidence

    def get_possible_lattices(self):
      return self._possible_lattices

  return DialsSymmetryWrapper()

if __name__ == '__main__':
  p = DialsSymmetry()

  hklin = sys.argv[1]

  p.set_hklin(hklin)

  p.decide_pointgroup()

  pass
<commit_msg>Fix for dials-full SymmetryWrapper error if symmetry > lattice symmetry<commit_after>from __future__ import absolute_import, division, print_function

import json
import math
import os
import shutil
import sys

from cctbx import sgtbx, crystal, uctbx
from cctbx.sgtbx import bravais_types

from xia2.Driver.DriverFactory import DriverFactory
from xia2.Handlers.Phil import PhilIndex
from xia2.Handlers.Streams import Chatter, Debug
from xia2.Handlers.Syminfo import Syminfo
# this was rather complicated - now simpler!
from xia2.lib.SymmetryLib import (clean_reindex_operator, lauegroup_to_lattice,
                                  spacegroup_name_xHM_to_old)
# XDS_ASCII meddling things
from xia2.Modules.XDS_ASCII import remove_misfits

def DialsSymmetry(DriverType = None):
  '''A factory for DialsSymmetryWrapper classes.'''

  DriverInstance = DriverFactory.Driver(DriverType)

  class DialsSymmetryWrapper(DriverInstance.__class__):
    '''A wrapper for dials.symmetry'''

    def __init__(self):
      # generic things
      super(DialsSymmetryWrapper, self).__init__()

      self.set_executable('dials.symmetry')

      self._input_laue_group = None

      self._experiments_filenames = []
      self._reflections_filenames = []
      self._output_experiments_filename = None
      self._output_reflections_filename = None

      self._hklin = None
      self._hklout = None
      self._pointgroup = None
      self._spacegroup = None
      self._reindex_matrix = None
      self._reindex_operator = None
      self._spacegroup_reindex_matrix = None
      self._spacegroup_reindex_operator = None
      self._confidence = 0.0
      self._hklref = None
      self._xdsin = None
      self._probably_twinned = False
      self._allow_out_of_sequence_files = False

      self._relative_length_tolerance = 0.05
      self._absolute_angle_tolerance = 2

      # space to store all possible solutions, to allow discussion of
      # the correct lattice with the indexer... this should be a
      # list containing e.g. 'tP'
      self._possible_lattices = []

      self._lattice_to_laue = { }

      # all "likely" spacegroups...
      self._likely_spacegroups = []

      # and unit cell information
      self._cell_info = { }
      self._cell = None

      self._json = None

    #def set_hklref(self, hklref):
      #self._hklref = hklref

    def set_hklin(self, hklin):
      self._hklin = hklin

    def get_hklin(self):
      return self._hklin

    def add_experiments(self, experiments):
      self._experiments_filenames.append(experiments)

    def add_reflections(self, reflections):
      self._reflections_filenames.append(reflections)

    def set_experiments_filename(self, experiments_filename):
      self._experiments_filenames = [experiments_filename]

    def set_reflections_filename(self, reflections_filename):
      self._reflections_filenames = [reflections_filename]

    def set_output_experiments_filename(self, experiments_filename):
      self._output_experiments_filename = experiments_filename

    def set_output_reflections_filename(self, reflections_filename):
      self._output_reflections_filename = reflections_filename

    def get_output_reflections_filename(self):
      return self._output_reflections_filename

    def get_output_experiments_filename(self):
      return self._output_experiments_filename

    def set_json(self, json):
      self._json = json

    def set_allow_out_of_sequence_files(self, allow=True):
      self._allow_out_of_sequence_files = allow

    def set_tolerance(self, relative_length_tolerance=0.05, absolute_angle_tolerance=2):
      self._relative_length_tolerance=relative_length_tolerance
      self._absolute_angle_tolerance=absolute_angle_tolerance

    #def get_hklref(self):
      #return self._hklref

    #def check_hklref(self):
      #if self._hklref is None:
        #raise RuntimeError('hklref not defined')
      #if not os.path.exists(self._hklref):
        #raise RuntimeError('hklref %s does not exist' % self._hklref)

    def set_correct_lattice(self, lattice):
      '''In a rerunning situation, set the correct lattice, which will
      assert a correct lauegroup based on the previous run of the
      program...'''

      if self._lattice_to_laue == { }:
        raise RuntimeError('no lattice to lauegroup mapping')

      if lattice not in self._lattice_to_laue:
        raise RuntimeError('lattice %s not possible' % lattice)

      self._input_laue_group = self._lattice_to_laue[lattice]

    def decide_pointgroup(self, ignore_errors=False, batches=None):
      '''Decide on the correct pointgroup for hklin.'''

      self.clear_command_line()

      if self._hklref:
        self.add_command_line('hklref')
        self.add_command_line(self._hklref)

      if self._hklin is not None:
        assert os.path.isfile(self._hklin)
        self.add_command_line("'%s'" % self._hklin)
      else:
        assert self._experiments_filenames# is not None
        assert self._reflections_filenames# is not None
        for exp in self._experiments_filenames:
          self.add_command_line("'%s'" % exp)
        for refl in self._reflections_filenames:
          self.add_command_line("'%s'" % refl)

        if not self._output_experiments_filename:
          self._output_experiments_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_experiments.json' % self.get_xpid())
        if not self._output_reflections_filename:
          self._output_reflections_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_reflections.pickle' % self.get_xpid())

        self.add_command_line("output.experiments='%s'" % self._output_experiments_filename)
        self.add_command_line("output.reflections='%s'" % self._output_reflections_filename)

      self.add_command_line('relative_length_tolerance=%s' % self._relative_length_tolerance)
      self.add_command_line('absolute_angle_tolerance=%s' % self._absolute_angle_tolerance)
      if not self._json:
        self._json = os.path.join(self.get_working_directory(),
                                '%d_dials_symmetry.json' % self.get_xpid())

      self.add_command_line("output.json='%s'" % self._json)

      self.start()

      self.close_wait()

      # check for errors
      self.check_for_errors()

      output = self.get_all_output()

      assert os.path.exists(self._json)
      with open(self._json, 'rb') as f:
        d = json.load(f)

      best_solution = d['subgroup_scores'][0]
      patterson_group = sgtbx.space_group(
        str(best_solution['patterson_group']))
      if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
        patterson_group = patterson_group.build_derived_acentric_group()

      cb_op_inp_min = sgtbx.change_of_basis_op(str(d['cb_op_inp_min']))
      cb_op_min_best = sgtbx.change_of_basis_op(str(best_solution['cb_op']))

      input_cell = uctbx.unit_cell(d['input_symmetry']['unit_cell'])
      cb_op_inp_best = cb_op_min_best * cb_op_inp_min
      best_cell = input_cell.change_basis(
        cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_best)))
      # ^^ not necessarily in reference setting e.g I2/m not C2/m

      cs = crystal.symmetry(unit_cell=best_cell,
        space_group=patterson_group, assert_is_compatible_unit_cell=False)
      cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting()
      cs_reference = cs.as_reference_setting()
      self._pointgroup = cs_reference.space_group().type().lookup_symbol()

      self._confidence = best_solution['confidence']
      self._totalprob = best_solution['likelihood']
      cb_op_inp_best = cb_op_inp_best * cb_op_best_to_ref
      self._reindex_operator = cb_op_inp_best.as_xyz()
      self._reindex_matrix = cb_op_inp_best.c().r().as_double()

      if not self._input_laue_group and not self._hklref:
        for score in d['subgroup_scores']:
          patterson_group = sgtbx.space_group(str(score['patterson_group']))
          if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
            patterson_group = patterson_group.build_derived_acentric_group()

          cb_op_min_this = sgtbx.change_of_basis_op(str(score['cb_op']))
          cb_op_inp_this = cb_op_min_this * cb_op_inp_min
          unit_cell = input_cell.change_basis(
            cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_this)))
          cs = crystal.symmetry(unit_cell=unit_cell,
            space_group=patterson_group, assert_is_compatible_unit_cell=False)
          cs_reference = cs.as_reference_setting()
          patterson_group = cs_reference.space_group()

          netzc = score['z_cc_net']
          # record this as a possible lattice if its Z score is positive
          lattice = str(bravais_types.bravais_lattice(
            group=patterson_group))
          if not lattice in self._possible_lattices:
            if netzc > 0.0:
              self._possible_lattices.append(lattice)
            self._lattice_to_laue[lattice] = patterson_group.type().lookup_symbol()
          self._likely_spacegroups.append(patterson_group.type().lookup_symbol())
      return

    def get_reindex_matrix(self):
      return self._reindex_matrix

    def get_reindex_operator(self):
      return self._reindex_operator

    def get_pointgroup(self):
      return self._pointgroup

    def get_cell(self):
      return self._cell

    def get_probably_twinned(self):
      return self._probably_twinned

    #def get_spacegroup(self):
      #return self._spacegroup

    #def get_spacegroup_reindex_operator(self):
      #return self._spacegroup_reindex_operator

    #def get_spacegroup_reindex_matrix(self):
      #return self._spacegroup_reindex_matrix

    # FIXME spacegroup != pointgroup
    decide_spacegroup = decide_pointgroup
    get_spacegroup = get_pointgroup
    get_spacegroup_reindex_operator = get_reindex_operator
    get_spacegroup_reindex_matrix = get_reindex_matrix

    def get_likely_spacegroups(self):
      return self._likely_spacegroups

    def get_confidence(self):
      return self._confidence

    def get_possible_lattices(self):
      return self._possible_lattices

  return DialsSymmetryWrapper()

if __name__ == '__main__':
  p = DialsSymmetry()

  hklin = sys.argv[1]

  p.set_hklin(hklin)

  p.decide_pointgroup()

  pass
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

import os
import errno
import csv

import json

from collections import OrderedDict

########################################################################
# Public general file util methods
########################################################################

def mkdir_p(path):
    # Function performs cli command: mkdir -p
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

########################################################################
# Read methods
########################################################################

def read(file_path, **kwargs):
    return(read_file(file_path, **kwargs))

def read_file(file_path, **kwargs):
    file_ext = os.path.splitext(file_path.upper())[1]

    if 'delimiter' in kwargs:
        delimiter = kwargs['delimiter']
    else:
        delimiter = ','

    if 'headings' in kwargs:
        headings = kwargs['headings']
    else:
        headings = False

    if 'JSON' in file_ext:
        return read_json(file_path)
    elif 'CSV' in file_ext:
        return read_csv(file_path, delimiter, headings, **kwargs)
    else:
        with open(file_path) as fp:
            return fp.read()

def read_json(file_path, encoding='UTF-8', ordered=True):
    kwargs = {}
    return_dict = {}
    if os.path.exists(file_path):
        kwargs['encoding'] = encoding
        if ordered:
            kwargs['object_pairs_hook'] = OrderedDict
        with open(file_path, 'r') as fp:
            return_dict = json.load(fp, **kwargs)
    return return_dict

def read_csv(file_path, delimiter=',', headings=False, **kwargs):
    # [{'r0c1': 'r1c1', 'r0c2': 'r1c2', ... , 'r0cn': 'r1cn'}]

    return_list = []
    with open(file_path, 'r') as fp:
        row = 0
        if 'heads_list' in kwargs:
            heads = kwargs['heads_list']
        else:
            heads = []
        # lines = fp.readlines()
        for line_in in fp:
            entry_dict = {}
            line = line_in.rstrip('\n').split(delimiter)
            if row <= 0 and headings and not heads:
                heads = line
            else:
                col = 0
                for entry in line:
                    if heads:
                        head = heads[col]
                    else:
                        head = col
                    entry_dict[head] = entry
                    col += 1
                return_list.append(entry_dict)
            row += 1
    return return_list

########################################################################
# Write methods
########################################################################

def write_file(file_path, data, mode='w'):
    with open(file_path, mode) as fp:
        file_ext = os.path.splitext(file_path.upper())[1]
        if 'JSON' in file_ext:
            json.dump(data, fp, indent=4, sort_keys=True)
        elif 'CSV' in file_ext:
            contents = ''
            if isinstance(data, dict):
                for key, vals in data.items():
                    new_line = key + ',' + ','.join(vals)
                    contents = '\n'.join([contents, new_line])
            elif isinstance(data, list):
                contents = '\n'.join(data)
            fp.write(contents)
        else:
            fp.write(data)

def write_csv(file_path, rows):
    with open(file_path, 'w') as fp:
        writer = csv.DictWriter(fp, fieldnames=rows[0].keys())
        writer.writeheader()
        for row in rows:
            stripped_row = {
                k: (
                    v.strip()
                    if isinstance(v, basestring)
                    else v
                )
                for k, v in row.iteritems()
            }
            writer.writerow(stripped_row)

########################################################################
# Data format methods
########################################################################

def get_dict_value(*key_list, **key_dict):
    for key in key_list:
        if key in key_dict:
            return key_dict[key]

def format_list_of_dicts_from_file(in_dict_list, base_path):
    base_dict = read_json(base_path)
    return format_list_of_dicts(in_dict_list, base_dict)

def format_dictionary_from_file(in_dict, base_path):
    base_dict = read_json(base_path)
    return format_dictionary(in_dict, base_dict)

def format_dictionary(in_dict, base_dict):
    return_dict = {}

    for key, val in base_dict.iteritems():
        if '$' in key:
            key = key[1:]
            new_key = in_dict[key]
        else:
            new_key = key

        if isinstance(val, dict):
            new_val = format_dictionary(in_dict[key], val)
        return_dict[new_key] = new_val
    return return_dict

def format_list_of_dicts(in_dict_list, base_dict):
    return_list = []

    for in_dict in in_dict_list:
        return_list.append(format_dictionary(in_dict, base_dict))
    return return_list

########################################################################
# Conversion methods
########################################################################

def convert_file(in_path, out_path, out_write=True, **kwargs):
    in_ext = os.path.splitext(in_path)[1].upper()
    # out_ext = os.path.splitext(out_path)[1].upper()

    if 'template' in kwargs:
        template = kwargs['template']
    else:
        template = None

    # if 'out_format' in kwargs:
    #     out_format = kwargs['out_format']
    # else:
    #     out_format = False

    if 'in_delimiter' in kwargs:
        in_delim = kwargs['in_delimiter']
    else:
        in_delim = ','

    # if 'out_delimiter' in kwargs:
    #     out_delim = kwargs['out_delimiter']
    # else:
    #     out_delim = ','

    if 'in_heads_list' in kwargs:
        in_heads_list = kwargs['in_heads_list']
        in_has_heads = True
    else:
        in_heads_list = None
        in_has_heads = False

    # if 'out_heads_list' in kwargs:
    #     out_heads_list = kwargs['out_heads_list']
    #     out_has_heads = True
    # else:
    #     out_heads_list = None
    #     out_has_heads = False

    in_data = None
    out_data = None

    if 'CSV' in in_ext or 'TXT' in in_ext:
        in_data = read_csv(in_path, in_delim, in_has_heads, **{
            'heads_list': in_heads_list,
        })
    elif 'JSON' in in_ext:
        in_data = read_json(in_path)
    else:
        kwargs['heads_list'] = in_heads_list
        in_data = read_file(in_path, **kwargs)

    if template and in_data:
        out_data = format_dictionary(in_data, template)
    else:
        out_data = in_data

    # if out_format:
    #     out_data = format_csp(out_data)

    if out_write and out_data:
        write_file(out_path, out_data)
    return out_data
<commit_msg>Added heading prepending and delimiter conversion methods<commit_after>from __future__ import absolute_import, print_function

import os
import errno
import csv
import fileinput
import json

from collections import OrderedDict

########################################################################
# Public general file util methods
########################################################################

def mkdir_p(path):
    # Function performs cli command: mkdir -p
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

########################################################################
# Read methods
########################################################################

def read(file_path, **kwargs):
    return read_file(file_path, **kwargs)

def read_file(file_path, **kwargs):
    file_ext = os.path.splitext(file_path.upper())[1]

    if 'delimiter' in kwargs:
        delimiter = kwargs['delimiter']
    else:
        delimiter = ','

    if 'headings' in kwargs:
        headings = kwargs['headings']
    else:
        headings = False

    if 'JSON' in file_ext:
        return read_json(file_path)
    elif 'CSV' in file_ext:
        return read_csv(file_path, delimiter, headings, **kwargs)
    else:
        with open(file_path) as fp:
            return fp.read()

def read_json(file_path, encoding='UTF-8', ordered=True):
    kwargs = {}
    return_dict = {}
    if os.path.exists(file_path):
        kwargs['encoding'] = encoding
        if ordered:
            kwargs['object_pairs_hook'] = OrderedDict
        with open(file_path, 'r') as fp:
            return_dict = json.load(fp, **kwargs)
    return return_dict

def read_csv(file_path, delimiter=',', headings=False, **kwargs):
    # [{'r0c1': 'r1c1', 'r0c2': 'r1c2', ... , 'r0cn': 'r1cn'}]

    return_list = []
    with open(file_path, 'r') as fp:
        row = 0
        if 'heads_list' in kwargs:
            heads = kwargs['heads_list']
        else:
            heads = []
        # lines = fp.readlines()
        for line_in in fp:
            entry_dict = {}
            line = line_in.rstrip('\n').split(delimiter)
            if row <= 0 and headings and not heads:
                heads = line
            else:
                col = 0
                for entry in line:
                    if heads:
                        head = heads[col]
                    else:
                        head = col
                    entry_dict[head] = entry
                    col += 1
                return_list.append(entry_dict)
            row += 1
    return return_list

########################################################################
# Write methods
########################################################################

def write_file(file_path, data, mode='w'):
    with open(file_path, mode) as fp:
        file_ext = os.path.splitext(file_path.upper())[1]
        if 'JSON' in file_ext:
            json.dump(data, fp, indent=4, sort_keys=True)
        elif 'CSV' in file_ext:
            contents = ''
            if isinstance(data, dict):
                for key, vals in data.items():
                    new_line = key + ',' + ','.join(vals)
                    contents = '\n'.join([contents, new_line])
            elif isinstance(data, list):
                contents = '\n'.join(data)
            fp.write(contents)
        else:
            fp.write(data)

def write_csv(file_path, rows):
    with open(file_path, 'w') as fp:
        writer = csv.DictWriter(fp, fieldnames=rows[0].keys())
        writer.writeheader()
        for row in rows:
            stripped_row = {
                k: (
                    v.strip()
                    if isinstance(v, str)
                    else v
                )
                for k, v in row.items()
            }
            writer.writerow(stripped_row)

########################################################################
# Data format methods
########################################################################

def get_dict_value(*key_list, **key_dict):
    for key in key_list:
        if key in key_dict:
            return key_dict[key]

def format_list_of_dicts_from_file(in_dict_list, base_path):
    base_dict = read_json(base_path)
    return format_list_of_dicts(in_dict_list, base_dict)

def format_dictionary_from_file(in_dict, base_path):
    base_dict = read_json(base_path)
    return format_dictionary(in_dict, base_dict)

def format_dictionary(in_dict, base_dict):
    return_dict = {}

    for key, val in base_dict.items():
        if '$' in key:
            key = key[1:]
            new_key = in_dict[key]
        else:
            new_key = key

        if isinstance(val, dict):
            new_val = format_dictionary(in_dict[key], val)
        return_dict[new_key] = new_val
    return return_dict

def format_list_of_dicts(in_dict_list, base_dict):
    return_list = []

    for in_dict in in_dict_list:
        return_list.append(format_dictionary(in_dict, base_dict))
    return return_list

def batch_prepend_headings(basedir, head_list, delim=','):
    for base_file in os.listdir(basedir):
        prepend_headings(os.path.join(basedir, base_file), head_list, delim)

def prepend_headings(base_file, head_list, delim=','):
    headers = delim.join(head_list)
    if os.path.exists(base_file):
        for line in fileinput.input(files=[base_file], inplace=True):
            if fileinput.isfirstline() and headers not in line:
                print(headers)

def convert_delimiter_inline(base_file, old_delimiter=',', new_delimiter='|'):
    if os.path.exists(base_file):
        for line in fileinput.input(files=[base_file], inplace=True):
            print(line[:-1].replace(old_delimiter, new_delimiter))

########################################################################
# Conversion methods
########################################################################

def convert_file(in_path, out_path, out_write=True, **kwargs):
    in_ext = os.path.splitext(in_path)[1].upper()
    # out_ext = os.path.splitext(out_path)[1].upper()

    if 'template' in kwargs:
        template = kwargs['template']
    else:
        template = None

    # if 'out_format' in kwargs:
    #     out_format = kwargs['out_format']
    # else:
    #     out_format = False

    if 'in_delimiter' in kwargs:
        in_delim = kwargs['in_delimiter']
    else:
        in_delim = ','

    # if 'out_delimiter' in kwargs:
    #     out_delim = kwargs['out_delimiter']
    # else:
    #     out_delim = ','

    if 'in_heads_list' in kwargs:
        in_heads_list = kwargs['in_heads_list']
        in_has_heads = True
    else:
        in_heads_list = None
        in_has_heads = False

    # if 'out_heads_list' in kwargs:
    #     out_heads_list = kwargs['out_heads_list']
    #     out_has_heads = True
    # else:
    #     out_heads_list = None
    #     out_has_heads = False

    in_data = None
    out_data = None

    if 'CSV' in in_ext or 'TXT' in in_ext:
        in_data = read_csv(in_path, in_delim, in_has_heads, **{
            'heads_list': in_heads_list,
        })
    elif 'JSON' in in_ext:
        in_data = read_json(in_path)
    else:
        kwargs['heads_list'] = in_heads_list
        in_data = read_file(in_path, **kwargs)

    if template and in_data:
        out_data = format_dictionary(in_data, template)
    else:
        out_data = in_data

    # if out_format:
    #     out_data = format_csp(out_data)

    if out_write and out_data:
        write_file(out_path, out_data)
    return out_data
<|endoftext|>


<commit_before>from amqp import AMQPError
from redis import RedisError

from django.conf import settings

from libs.redis_db import RedisToStream
from libs.services import Service
from polyaxon.celery_api import app as celery_app
from polyaxon.settings import EventsCeleryTasks, RoutingKeys

class PublisherService(Service):
    __all__ = ('publish_experiment_log', 'publish_build_job_log', 'setup')

    def __init__(self):
        self._logger = None

    def publish_experiment_log(self,
                               log_line,
                               status,
                               experiment_uuid,
                               experiment_name,
                               job_uuid,
                               task_type=None,
                               task_idx=None):
        try:
            log_line = log_line.decode('utf-8')
        except AttributeError:
            pass

        self._logger.info("Publishing log event for task: %s.%s, %s", task_type, task_idx,
                          experiment_name)
        celery_app.send_task(
            EventsCeleryTasks.EVENTS_HANDLE_LOGS_SIDECAR,
            kwargs={
                'experiment_name': experiment_name,
                'experiment_uuid': experiment_uuid,
                'job_uuid': job_uuid,
                'log_line': log_line,
                'task_type': task_type,
                'task_idx': task_idx})
        try:
            should_stream = (RedisToStream.is_monitored_job_logs(job_uuid) or
                             RedisToStream.is_monitored_experiment_logs(experiment_uuid))
        except RedisError:
            should_stream = False
        if should_stream:
            self._logger.info("Streaming new log event for experiment: %s", experiment_uuid)

            with celery_app.producer_or_acquire(None) as producer:
                try:
                    producer.publish(
                        {
                            'experiment_uuid': experiment_uuid,
                            'job_uuid': job_uuid,
                            'log_line': log_line,
                            'status': status,
                            'task_type': task_type,
                            'task_idx': task_idx
                        },
                        routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS,
                                                      experiment_uuid,
                                                      job_uuid),
                        exchange=settings.INTERNAL_EXCHANGE,
                    )
                except (TimeoutError, AMQPError):
                    pass

    def publish_build_job_log(self, log_line, job_uuid, job_name):
        try:
            log_line = log_line.decode('utf-8')
        except AttributeError:
            pass

        self._logger.info("Publishing log event for task: %s", job_uuid)
        celery_app.send_task(
            EventsCeleryTasks.EVENTS_HANDLE_LOGS_BUILD_JOB,
            kwargs={'job_uuid': job_uuid, 'job_name': job_name, 'log_line': log_line})

    def setup(self):
        import logging

        self._logger = logging.getLogger('polyaxon.monitors.publisher')
<commit_msg>Add jobs logs publisher<commit_after>from amqp import AMQPError
from redis import RedisError

from django.conf import settings

from libs.redis_db import RedisToStream
from libs.services import Service
from polyaxon.celery_api import app as celery_app
from polyaxon.settings import EventsCeleryTasks, RoutingKeys

class PublisherService(Service):
    __all__ = ('publish_experiment_log', 'publish_build_job_log', 'publish_job_log', 'setup')

    def __init__(self):
        self._logger = None

    def publish_experiment_log(self,
                               log_line,
                               status,
                               experiment_uuid,
                               experiment_name,
                               job_uuid,
                               task_type=None,
                               task_idx=None):
        try:
            log_line = log_line.decode('utf-8')
        except AttributeError:
            pass

        self._logger.info("Publishing log event for task: %s.%s, %s", task_type, task_idx,
                          experiment_name)
        celery_app.send_task(
            EventsCeleryTasks.EVENTS_HANDLE_LOGS_EXPERIMENT_SIDECAR,
            kwargs={
                'experiment_name': experiment_name,
                'experiment_uuid': experiment_uuid,
                'job_uuid': job_uuid,
                'log_line': log_line,
                'task_type': task_type,
                'task_idx': task_idx})
        try:
            should_stream = (RedisToStream.is_monitored_job_logs(job_uuid) or
                             RedisToStream.is_monitored_experiment_logs(experiment_uuid))
        except RedisError:
            should_stream = False
        if should_stream:
            self._logger.info("Streaming new log event for experiment: %s", experiment_uuid)

            with celery_app.producer_or_acquire(None) as producer:
                try:
                    producer.publish(
                        {
                            'experiment_uuid': experiment_uuid,
                            'job_uuid': job_uuid,
                            'log_line': log_line,
                            'status': status,
                            'task_type': task_type,
                            'task_idx': task_idx
                        },
                        routing_key='{}.{}.{}'.format(RoutingKeys.LOGS_SIDECARS,
                                                      experiment_uuid,
                                                      job_uuid),
                        exchange=settings.INTERNAL_EXCHANGE,
                    )
                except (TimeoutError, AMQPError):
                    pass

    def publish_build_job_log(self, log_line, job_uuid, job_name):
        try:
            log_line = log_line.decode('utf-8')
        except AttributeError:
            pass

        self._logger.info("Publishing log event for task: %s", job_uuid)
        celery_app.send_task(
            EventsCeleryTasks.EVENTS_HANDLE_LOGS_BUILD_JOB,
            kwargs={'job_uuid': job_uuid, 'job_name': job_name, 'log_line': log_line})

    def publish_job_log(self, log_line, job_uuid, job_name):
        try:
            log_line = log_line.decode('utf-8')
        except AttributeError:
            pass

        self._logger.info("Publishing log event for task: %s", job_uuid)
        celery_app.send_task(
            EventsCeleryTasks.EVENTS_HANDLE_LOGS_JOB_SIDECAR,
            kwargs={'job_uuid': job_uuid, 'job_name': job_name, 'log_line': log_line})

    def setup(self):
        import logging

        self._logger = logging.getLogger('polyaxon.monitors.publisher')
<|endoftext|>