<commit_before>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Organization Admins.
"""

__authors__ = [
    '"Lennard de Rijk" <ljvderijk@gmail.com>'
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import forms
from django.utils.translation import ugettext

from soc.logic import dicts
from soc.logic.models import organization as org_logic
from soc.logic.models import org_admin as org_admin_logic
from soc.logic.models import org_app as org_app_logic
from soc.views.helper import access
from soc.views.helper import decorators
from soc.views.helper import dynaform
from soc.views.helper import params as params_helper
from soc.views.helper import widgets
from soc.views.models import organization as org_view
from soc.views.models import role

import soc.logic.models.org_admin

class View(role.View):
  """View methods for the Organization Admin model.
  """

  DEF_ALREADY_AGREED_MSG = ugettext(
      "You have already accepted this agreement when submitting "
      "the organization application.")

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = access.Checker(params)
    rights['create'] = ['checkIsDeveloper']
    rights['edit'] = [('checkHasActiveRoleForScope', org_admin_logic.logic)]
    rights['delete'] = ['checkIsDeveloper']
    rights['invite'] = [('checkHasActiveRoleForScope',
                         org_admin_logic.logic)]
    rights['accept_invite'] = [('checkCanCreateFromRequest', 'org_admin'),
        'checkIsNotStudentForProgramOfOrg']
    rights['process_request'] = [
        ('checkHasActiveRoleForScope', org_admin_logic.logic),
        ('checkCanProcessRequest', 'org_admin')]
    rights['manage'] = [
        ('checkIsAllowedToManageRole', [org_admin_logic.logic,
             org_admin_logic.logic])]

    new_params = {}
    new_params['logic'] = soc.logic.models.org_admin.logic
    new_params['group_logic'] = org_logic.logic
    new_params['group_view'] = org_view.view
    new_params['rights'] = rights

    new_params['scope_view'] = org_view

    new_params['name'] = "Organization Admin"
    new_params['module_name'] = "org_admin"
    new_params['sidebar_grouping'] = 'Organizations'

    new_params['extra_dynaexclude'] = ['agreed_to_tos', 'program']

    new_params['create_dynafields'] = [
        {'name': 'scope_path',
         'base': forms.fields.CharField,
         'widget': forms.HiddenInput,
         'required': True,
         },
        {'name': 'admin_agreement',
         'base': forms.fields.CharField,
         'required': False,
         'widget': widgets.AgreementField,
         'group': ugettext("5. Terms of Service"),
         },
        {'name': 'agreed_to_admin_agreement',
         'base': forms.fields.BooleanField,
         'initial': False,
         'required':True,
         'label': ugettext('I agree to the Admin Agreement'),
         'group': ugettext("5. Terms of Service"),
         },
        ]

    new_params['allow_invites'] = True
    new_params['show_in_roles_overview'] = True

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

    # register the role with the group_view
    params['group_view'].registerRole(params['module_name'], self)

    # create and store the special form for invited users
    dynafields = [
        {'name': 'link_id',
         'base': forms.CharField,
         'widget': widgets.ReadOnlyInput(),
         'required': False,
         },
        {'name': 'admin_agreement',
         'base': forms.fields.Field,
         'required': False,
         'widget': widgets.AgreementField,
         'group': ugettext("5. Terms of Service"),
        },
        ]

    dynaproperties = params_helper.getDynaFields(dynafields)

    invited_create_form = dynaform.extendDynaForm(
        dynaform = self._params['create_form'],
        dynaproperties = dynaproperties)

    params['invited_create_form'] = invited_create_form

  def _editPost(self, request, entity, fields):
    """See base.View._editPost().
    """

    if not entity:
      fields['user'] = fields['link_id']
      fields['link_id'] = fields['user'].link_id
      group_logic = params['group_logic']
      group_entity = group_logic.getFromKeyName(fields['scope_path'])
      fields['program'] = group_entity.scope

    fields['agreed_to_tos'] = fields['agreed_to_admin_agreement']

    super(View, self)._editPost(request, entity, fields)

  def _acceptInvitePost(self, fields, request, context, params, **kwargs):
    """Fills in the fields that were missing in the invited_created_form.

    For params see base.View._acceptInvitePost()
    """

    # fill in the appropriate fields that were missing in the form
    fields['user'] = fields['link_id']
    fields['link_id'] = fields['user'].link_id
    fields['agreed_to_tos'] = fields['agreed_to_admin_agreement']

    group_logic = params['group_logic']
    group_entity = group_logic.getFromKeyName(fields['scope_path'])
    fields['program'] = group_entity.scope

  def _editGet(self, request, entity, form):
    """Sets the content of the agreed_to_tos_on field and replaces.

    Also replaces the agreed_to_tos field with a hidden field when the ToS has been signed.
    For params see base.View._editGet().
    """

    if entity.agreed_to_tos:
      form.fields['agreed_to_admin_agreement'] = forms.fields.BooleanField(
          widget=forms.HiddenInput, initial=entity.agreed_to_tos,
          required=True)

    super(View, self)._editGet(request, entity, form)

  def _editContext(self, request, context):
    """See base.View._editContext.
    """

    entity = context['entity']
    form = context['form']

    if 'scope_path' in form.initial:
      scope_path = form.initial['scope_path']
    elif 'scope_path' in request.POST:
      # TODO: do this nicely
      scope_path = request.POST['scope_path']
    else:
      # TODO: is this always sufficient?
      form.fields['admin_agreement'] = None
      return

    org_app = org_app_logic.logic.getFromKeyName(scope_path)

    if not entity and org_app:
      if org_app.applicant.key() == context['user'].key():
        form.fields['agreed_to_admin_agreement'] = forms.fields.BooleanField(
            widget=widgets.ReadOnlyInput, initial=True, required=True,
            help_text=self.DEF_ALREADY_AGREED_MSG)

    entity = org_logic.logic.getFromKeyName(scope_path)

    if not (entity and entity.scope and entity.scope.org_admin_agreement):
      return

    content = entity.scope.org_admin_agreement.content

    form.fields['admin_agreement'].widget.text = content

view = View()

accept_invite = decorators.view(view.acceptInvite)
admin = decorators.view(view.admin)
create = decorators.view(view.create)
delete = decorators.view(view.delete)
edit = decorators.view(view.edit)
invite = decorators.view(view.invite)
list = decorators.view(view.list)
manage = decorators.view(view.manage)
process_request = decorators.view(view.processRequest)
public = decorators.view(view.public)
export = decorators.view(view.export)
<commit_msg>Fix the bug where undefined variable was used in _editPost method of soc.view.models.org_admin module.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Organization Admins.
"""

__authors__ = [
    '"Lennard de Rijk" <ljvderijk@gmail.com>'
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import forms
from django.utils.translation import ugettext

from soc.logic import dicts
from soc.logic.models import organization as org_logic
from soc.logic.models import org_admin as org_admin_logic
from soc.logic.models import org_app as org_app_logic
from soc.views.helper import access
from soc.views.helper import decorators
from soc.views.helper import dynaform
from soc.views.helper import params as params_helper
from soc.views.helper import widgets
from soc.views.models import organization as org_view
from soc.views.models import role

import soc.logic.models.org_admin

class View(role.View):
  """View methods for the Organization Admin model.
  """

  DEF_ALREADY_AGREED_MSG = ugettext(
      "You have already accepted this agreement when submitting "
      "the organization application.")

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = access.Checker(params)
    rights['create'] = ['checkIsDeveloper']
    rights['edit'] = [('checkHasActiveRoleForScope', org_admin_logic.logic)]
    rights['delete'] = ['checkIsDeveloper']
    rights['invite'] = [('checkHasActiveRoleForScope',
                         org_admin_logic.logic)]
    rights['accept_invite'] = [('checkCanCreateFromRequest', 'org_admin'),
        'checkIsNotStudentForProgramOfOrg']
    rights['process_request'] = [
        ('checkHasActiveRoleForScope', org_admin_logic.logic),
        ('checkCanProcessRequest', 'org_admin')]
    rights['manage'] = [
        ('checkIsAllowedToManageRole', [org_admin_logic.logic,
             org_admin_logic.logic])]

    new_params = {}
    new_params['logic'] = soc.logic.models.org_admin.logic
    new_params['group_logic'] = org_logic.logic
    new_params['group_view'] = org_view.view
    new_params['rights'] = rights

    new_params['scope_view'] = org_view

    new_params['name'] = "Organization Admin"
    new_params['module_name'] = "org_admin"
    new_params['sidebar_grouping'] = 'Organizations'

    new_params['extra_dynaexclude'] = ['agreed_to_tos', 'program']

    new_params['create_dynafields'] = [
        {'name': 'scope_path',
         'base': forms.fields.CharField,
         'widget': forms.HiddenInput,
         'required': True,
         },
        {'name': 'admin_agreement',
         'base': forms.fields.CharField,
         'required': False,
         'widget': widgets.AgreementField,
         'group': ugettext("5. Terms of Service"),
         },
        {'name': 'agreed_to_admin_agreement',
         'base': forms.fields.BooleanField,
         'initial': False,
         'required':True,
         'label': ugettext('I agree to the Admin Agreement'),
         'group': ugettext("5. Terms of Service"),
         },
        ]

    new_params['allow_invites'] = True
    new_params['show_in_roles_overview'] = True

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

    # register the role with the group_view
    params['group_view'].registerRole(params['module_name'], self)

    # create and store the special form for invited users
    dynafields = [
        {'name': 'link_id',
         'base': forms.CharField,
         'widget': widgets.ReadOnlyInput(),
         'required': False,
         },
        {'name': 'admin_agreement',
         'base': forms.fields.Field,
         'required': False,
         'widget': widgets.AgreementField,
         'group': ugettext("5. Terms of Service"),
        },
        ]

    dynaproperties = params_helper.getDynaFields(dynafields)

    invited_create_form = dynaform.extendDynaForm(
        dynaform = self._params['create_form'],
        dynaproperties = dynaproperties)

    params['invited_create_form'] = invited_create_form

  def _editPost(self, request, entity, fields):
    """See base.View._editPost().
    """

    if not entity:
      fields['user'] = fields['link_id']
      fields['link_id'] = fields['user'].link_id
      group_logic = self._params['group_logic']
      group_entity = group_logic.getFromKeyName(fields['scope_path'])
      fields['program'] = group_entity.scope

    fields['agreed_to_tos'] = fields['agreed_to_admin_agreement']

    super(View, self)._editPost(request, entity, fields)

  def _acceptInvitePost(self, fields, request, context, params, **kwargs):
    """Fills in the fields that were missing in the invited_created_form.

    For params see base.View._acceptInvitePost()
    """

    # fill in the appropriate fields that were missing in the form
    fields['user'] = fields['link_id']
    fields['link_id'] = fields['user'].link_id
    fields['agreed_to_tos'] = fields['agreed_to_admin_agreement']

    group_logic = params['group_logic']
    group_entity = group_logic.getFromKeyName(fields['scope_path'])
    fields['program'] = group_entity.scope

  def _editGet(self, request, entity, form):
    """Sets the content of the agreed_to_tos_on field and replaces.

    Also replaces the agreed_to_tos field with a hidden field when the ToS has been signed.
    For params see base.View._editGet().
    """

    if entity.agreed_to_tos:
      form.fields['agreed_to_admin_agreement'] = forms.fields.BooleanField(
          widget=forms.HiddenInput, initial=entity.agreed_to_tos,
          required=True)

    super(View, self)._editGet(request, entity, form)

  def _editContext(self, request, context):
    """See base.View._editContext.
    """

    entity = context['entity']
    form = context['form']

    if 'scope_path' in form.initial:
      scope_path = form.initial['scope_path']
    elif 'scope_path' in request.POST:
      # TODO: do this nicely
      scope_path = request.POST['scope_path']
    else:
      # TODO: is this always sufficient?
      form.fields['admin_agreement'] = None
      return

    org_app = org_app_logic.logic.getFromKeyName(scope_path)

    if not entity and org_app:
      if org_app.applicant.key() == context['user'].key():
        form.fields['agreed_to_admin_agreement'] = forms.fields.BooleanField(
            widget=widgets.ReadOnlyInput, initial=True, required=True,
            help_text=self.DEF_ALREADY_AGREED_MSG)

    entity = org_logic.logic.getFromKeyName(scope_path)

    if not (entity and entity.scope and entity.scope.org_admin_agreement):
      return

    content = entity.scope.org_admin_agreement.content

    form.fields['admin_agreement'].widget.text = content

view = View()

accept_invite = decorators.view(view.acceptInvite)
admin = decorators.view(view.admin)
create = decorators.view(view.create)
delete = decorators.view(view.delete)
edit = decorators.view(view.edit)
invite = decorators.view(view.invite)
list = decorators.view(view.list)
manage = decorators.view(view.manage)
process_request = decorators.view(view.processRequest)
public = decorators.view(view.public)
export = decorators.view(view.export)
<|endoftext|>


<commit_before># Copyright: (c) 2017, Allyson Bowles <@akatch>
# Copyright: (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = '''
    callback: unixy
    type: stdout
    author: Allyson Bowles <@akatch>
    short_description: condensed Ansible output
    version_added: 2.5
    description:
      - Consolidated Ansible output in the style of LINUX/UNIX startup logs.
    extends_documentation_fragment:
      - default_callback
    requirements:
      - set as stdout in configuration
'''

from os.path import basename

from ansible import constants as C
from ansible.module_utils._text import to_text
from ansible.plugins.callback import CallbackBase
from ansible.utils.color import colorize, hostcolor

class CallbackModule(CallbackBase):

    '''
    Design goals:
    - Print consolidated output that looks like a *NIX startup log
    - Defaults should avoid displaying unnecessary information wherever possible

    TODOs:
    - Only display task names if the task runs on at least one host
    - Add option to display all hostnames on a single line in the appropriate result color (failures may have a separate line)
    - Consolidate stats display
    - Display whether run is in --check mode
    - Don't show play name if no hosts found
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'unixy'

    def _run_is_verbose(self, result):
        return ((self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result)

    def _get_task_display_name(self, task):
        self.task_display_name = None
        display_name = task.get_name().strip().split(" : ")

        task_display_name = display_name[-1]
        if task_display_name.startswith("include"):
            return
        else:
            self.task_display_name = task_display_name

    def _preprocess_result(self, result):
        self.delegated_vars = result._result.get('_ansible_delegated_vars', None)
        self._handle_exception(result._result)
        self._handle_warnings(result._result)

    def _process_result_output(self, result, msg):
        task_host = result._host.get_name()
        task_result = "%s %s" % (task_host, msg)

        if self._run_is_verbose(result):
            task_result = "%s %s: %s" % (task_host, msg, self._dump_results(result._result, indent=4))

        if self.delegated_vars:
            task_delegate_host = self.delegated_vars['ansible_host']
            task_result = "%s -> %s %s" % (task_host, task_delegate_host, msg)

        if result._result.get('msg') and result._result.get('msg') != "All items completed":
            task_result += " | msg: " + to_text(result._result.get('msg'))

        if result._result.get('stdout'):
            task_result += " | stdout: " + result._result.get('stdout')

        if result._result.get('stderr'):
            task_result += " | stderr: " + result._result.get('stderr')

        return task_result

    def v2_playbook_on_task_start(self, task, is_conditional):
        self._get_task_display_name(task)
        if self.task_display_name is not None:
            self._display.display("%s..." % self.task_display_name)

    def v2_playbook_on_handler_task_start(self, task):
        self._get_task_display_name(task)
        if self.task_display_name is not None:
            self._display.display("%s (via handler)... " % self.task_display_name)

    def v2_playbook_on_play_start(self, play):
        # TODO display name of play and list of play hosts
        # TODO don't display play name if no hosts in play
        name = play.get_name().strip()
        if name:
            msg = u"\n- %s -" % name
        else:
            msg = u"---"

        self._display.display(msg)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        self._preprocess_result(result)
        display_color = C.COLOR_ERROR
        msg = "failed"

        task_result = self._process_result_output(result, msg)
        self._display.display("  " + task_result, display_color)

    def v2_runner_on_ok(self, result, msg="ok", display_color=C.COLOR_OK):
        self._preprocess_result(result)

        result_was_changed = ('changed' in result._result and result._result['changed'])
        if result_was_changed:
            msg = "done"
            display_color = C.COLOR_CHANGED

        task_result = self._process_result_output(result, msg)
        self._display.display("  " + task_result, display_color)

    def v2_runner_item_on_failed(self, result):
        self.v2_runner_on_failed(result)

    def v2_runner_on_unreachable(self, result):
        msg = "unreachable"
        display_color = C.COLOR_UNREACHABLE
        task_result = self._process_result_output(result, msg)

        self._display.display("  " + task_result, display_color)

    def v2_on_file_diff(self, result):
        if result._task.loop and 'results' in result._result:
            for res in result._result['results']:
                if 'diff' in res and res['diff'] and res.get('changed', False):
                    diff = self._get_diff(res['diff'])
                    if diff:
                        self._display.display(diff)
        elif 'diff' in result._result and result._result['diff'] and result._result.get('changed', False):
            diff = self._get_diff(result._result['diff'])
            if diff:
                self._display.display(diff)

    def v2_playbook_on_stats(self, stats):
        self._display.display("\n- Play recap -", screen_only=True)

        hosts = sorted(stats.processed.keys())
        for h in hosts:
            # TODO how else can we display these?
            t = stats.summarize(h)

            self._display.display(u"  %s : %s %s %s %s" % (
                hostcolor(h, t),
                colorize(u'ok', t['ok'], C.COLOR_OK),
                colorize(u'changed', t['changed'], C.COLOR_CHANGED),
                colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE),
                colorize(u'failed', t['failures'], C.COLOR_ERROR)),
                screen_only=True
            )

            self._display.display(u"  %s : %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize(u'ok', t['ok'], None),
                colorize(u'changed', t['changed'], None),
                colorize(u'unreachable', t['unreachable'], None),
                colorize(u'failed', t['failures'], None)),
                log_only=True
            )

    def v2_playbook_on_no_hosts_matched(self):
        self._display.display("  No hosts found!", color=C.COLOR_DEBUG)

    def v2_playbook_on_no_hosts_remaining(self):
        self._display.display("  Ran out of hosts!", color=C.COLOR_ERROR)

    def v2_playbook_on_start(self, playbook):
        # TODO display whether this run is happening in check mode
        self._display.display("Executing playbook %s" % basename(playbook._file_name))

        if self._display.verbosity > 3:
            if self._options is not None:
                for option in dir(self._options):
                    if option.startswith('_') or option in ['read_file', 'ensure_value', 'read_module']:
                        continue
                    val = getattr(self._options, option)
                    if val:
                        self._display.vvvv('%s: %s' % (option, val))

    def v2_runner_retry(self, result):
        msg = "  Retrying... (%d of %d)" % (result._result['attempts'], result._result['retries'])
        if (self._display.verbosity > 2 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result:
            msg += "Result was: %s" % self._dump_results(result._result)
        self._display.display(msg, color=C.COLOR_DEBUG)
<commit_msg>Do not append msg to verbose tasks (#46721)<commit_after># Copyright: (c) 2017, Allyson Bowles <@akatch>
# Copyright: (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = '''
    callback: unixy
    type: stdout
    author: Allyson Bowles <@akatch>
    short_description: condensed Ansible output
    version_added: 2.5
    description:
      - Consolidated Ansible output in the style of LINUX/UNIX startup logs.
    extends_documentation_fragment:
      - default_callback
    requirements:
      - set as stdout in configuration
'''

from os.path import basename

from ansible import constants as C
from ansible.module_utils._text import to_text
from ansible.plugins.callback import CallbackBase
from ansible.utils.color import colorize, hostcolor

class CallbackModule(CallbackBase):

    '''
    Design goals:
    - Print consolidated output that looks like a *NIX startup log
    - Defaults should avoid displaying unnecessary information wherever possible

    TODOs:
    - Only display task names if the task runs on at least one host
    - Add option to display all hostnames on a single line in the appropriate result color (failures may have a separate line)
    - Consolidate stats display
    - Display whether run is in --check mode
    - Don't show play name if no hosts found
    '''

    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'stdout'
    CALLBACK_NAME = 'unixy'

    def _run_is_verbose(self, result):
        return ((self._display.verbosity > 0 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result)

    def _get_task_display_name(self, task):
        self.task_display_name = None
        display_name = task.get_name().strip().split(" : ")

        task_display_name = display_name[-1]
        if task_display_name.startswith("include"):
            return
        else:
            self.task_display_name = task_display_name

    def _preprocess_result(self, result):
        self.delegated_vars = result._result.get('_ansible_delegated_vars', None)
        self._handle_exception(result._result)
        self._handle_warnings(result._result)

    def _process_result_output(self, result, msg):
        task_host = result._host.get_name()
        task_result = "%s %s" % (task_host, msg)

        if self._run_is_verbose(result):
            task_result = "%s %s: %s" % (task_host, msg, self._dump_results(result._result, indent=4))
            return task_result

        if self.delegated_vars:
            task_delegate_host = self.delegated_vars['ansible_host']
            task_result = "%s -> %s %s" % (task_host, task_delegate_host, msg)

        if result._result.get('msg') and result._result.get('msg') != "All items completed":
            task_result += " | msg: " + to_text(result._result.get('msg'))

        if result._result.get('stdout'):
            task_result += " | stdout: " + result._result.get('stdout')

        if result._result.get('stderr'):
            task_result += " | stderr: " + result._result.get('stderr')

        return task_result

    def v2_playbook_on_task_start(self, task, is_conditional):
        self._get_task_display_name(task)
        if self.task_display_name is not None:
            self._display.display("%s..." % self.task_display_name)

    def v2_playbook_on_handler_task_start(self, task):
        self._get_task_display_name(task)
        if self.task_display_name is not None:
            self._display.display("%s (via handler)... " % self.task_display_name)

    def v2_playbook_on_play_start(self, play):
        # TODO display name of play and list of play hosts
        # TODO don't display play name if no hosts in play
        name = play.get_name().strip()
        if name:
            msg = u"\n- %s -" % name
        else:
            msg = u"---"

        self._display.display(msg)

    def v2_runner_on_failed(self, result, ignore_errors=False):
        self._preprocess_result(result)
        display_color = C.COLOR_ERROR
        msg = "failed"

        task_result = self._process_result_output(result, msg)
        self._display.display("  " + task_result, display_color)

    def v2_runner_on_ok(self, result, msg="ok", display_color=C.COLOR_OK):
        self._preprocess_result(result)

        result_was_changed = ('changed' in result._result and result._result['changed'])
        if result_was_changed:
            msg = "done"
            display_color = C.COLOR_CHANGED

        task_result = self._process_result_output(result, msg)
        self._display.display("  " + task_result, display_color)

    def v2_runner_item_on_failed(self, result):
        self.v2_runner_on_failed(result)

    def v2_runner_on_unreachable(self, result):
        msg = "unreachable"
        display_color = C.COLOR_UNREACHABLE
        task_result = self._process_result_output(result, msg)

        self._display.display("  " + task_result, display_color)

    def v2_on_file_diff(self, result):
        if result._task.loop and 'results' in result._result:
            for res in result._result['results']:
                if 'diff' in res and res['diff'] and res.get('changed', False):
                    diff = self._get_diff(res['diff'])
                    if diff:
                        self._display.display(diff)
        elif 'diff' in result._result and result._result['diff'] and result._result.get('changed', False):
            diff = self._get_diff(result._result['diff'])
            if diff:
                self._display.display(diff)

    def v2_playbook_on_stats(self, stats):
        self._display.display("\n- Play recap -", screen_only=True)

        hosts = sorted(stats.processed.keys())
        for h in hosts:
            # TODO how else can we display these?
            t = stats.summarize(h)

            self._display.display(u"  %s : %s %s %s %s" % (
                hostcolor(h, t),
                colorize(u'ok', t['ok'], C.COLOR_OK),
                colorize(u'changed', t['changed'], C.COLOR_CHANGED),
                colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE),
                colorize(u'failed', t['failures'], C.COLOR_ERROR)),
                screen_only=True
            )

            self._display.display(u"  %s : %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize(u'ok', t['ok'], None),
                colorize(u'changed', t['changed'], None),
                colorize(u'unreachable', t['unreachable'], None),
                colorize(u'failed', t['failures'], None)),
                log_only=True
            )

    def v2_playbook_on_no_hosts_matched(self):
        self._display.display("  No hosts found!", color=C.COLOR_DEBUG)

    def v2_playbook_on_no_hosts_remaining(self):
        self._display.display("  Ran out of hosts!", color=C.COLOR_ERROR)

    def v2_playbook_on_start(self, playbook):
        # TODO display whether this run is happening in check mode
        self._display.display("Executing playbook %s" % basename(playbook._file_name))

        if self._display.verbosity > 3:
            if self._options is not None:
                for option in dir(self._options):
                    if option.startswith('_') or option in ['read_file', 'ensure_value', 'read_module']:
                        continue
                    val = getattr(self._options, option)
                    if val:
                        self._display.vvvv('%s: %s' % (option, val))

    def v2_runner_retry(self, result):
        msg = "  Retrying... (%d of %d)" % (result._result['attempts'], result._result['retries'])
        if (self._display.verbosity > 2 or '_ansible_verbose_always' in result._result) and '_ansible_verbose_override' not in result._result:
            msg += "Result was: %s" % self._dump_results(result._result)
        self._display.display(msg, color=C.COLOR_DEBUG)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# ndmg/preproc/rescale_bvec.py
# Created by Greg Kiar on 2016-02-12.
# Email: gkiar@jhu.edu

import numpy as np
import os.path as op

def rescale_bvec(bvec, bvec_new):
    """
    Normalizes b-vectors to be of unit length for the non-zero b-values. If the
    b-value is 0, the vector is untouched.

    Positional Arguments:
            - bvec:
                    File name of the original b-vectors file
            - bvec_new:
                    File name of the new (normalized) b-vectors file. Must have
                    extension `.bvec`
    """
    bv1 = np.array(np.loadtxt(bvec))
    print bv1
    # Enforce proper dimensions
    bv1 = bv1.T if bv1.shape[0] == 3 else bv1

    # Normalize values not close to norm 1
    bv2 = [b/np.linalg.norm(b) if not np.isclose(np.linalg.norm(b), 0)
           else b for b in bv1]
    print bv2

    try:
        assert(op.splitext(bvec_new)[1] == '.bvec')
        np.savetxt(bvec_new, bv2)
        pass
    except AssertionError:
        print 'Error: your new b-vector file must have extension .bvec to' +\
              ' be compatible with the the pipeline.'
        pass
    else:
        pass
    pass
<commit_msg>removed print statements leftover from debug<commit_after>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# ndmg/preproc/rescale_bvec.py
# Created by Greg Kiar on 2016-02-12.
# Email: gkiar@jhu.edu

import numpy as np
import os.path as op

def rescale_bvec(bvec, bvec_new):
    """
    Normalizes b-vectors to be of unit length for the non-zero b-values. If the
    b-value is 0, the vector is untouched.

    Positional Arguments:
            - bvec:
                    File name of the original b-vectors file
            - bvec_new:
                    File name of the new (normalized) b-vectors file. Must have
                    extension `.bvec`
    """
    bv1 = np.array(np.loadtxt(bvec))
    # Enforce proper dimensions
    bv1 = bv1.T if bv1.shape[0] == 3 else bv1

    # Normalize values not close to norm 1
    bv2 = [b/np.linalg.norm(b) if not np.isclose(np.linalg.norm(b), 0)
           else b for b in bv1]

    try:
        assert(op.splitext(bvec_new)[1] == '.bvec')
        np.savetxt(bvec_new, bv2)
        pass
    except AssertionError:
        print 'Error: your new b-vector file must have extension .bvec to' +\
              ' be compatible with the the pipeline.'
        pass
    else:
        pass
    pass
<|endoftext|>


<commit_before>import os

from sequana.misc import wget
from sequana import sequana_config_path
from easydev import DevTools, execute

class BuscoConfig(object):

    def __init__(self, species, outpath=None, sample_name=None, conda_bin_path=None,
            tmp_path=None, Rscript_bin_path=None):

        self.params = {}
        self.params['conda_bin_path'] = conda_bin_path # "/home/cokelaer/miniconda3/env/py3/bin"
        self.params['Rscript'] = Rscript_bin_path # "/home/cokelaer/miniconda3/bin"
        self.params['sample'] = sample_name  # prefix used in output files
        self.params['outpath'] = outpath  # where to store the output directory
        self.params['lineage_path'] = sequana_config_path + "/busco/{}".format(species)
        self.params['tmp_path'] = tmp_path

    def save_config_file(self, filename):
        from sequana import sequana_data
        config_generic = sequana_data("config.ini", "busco")
        data = open(config_generic, "r").read()
        data = data.format(**self.params)
        with open(filename, "w") as fh:
            fh.write(data)

class BuscoDownload(object):
    """

    This code downloads the BUSCO datasets (about G uncompressed). The data
    is stored in your config path (e.g. ~/.config/sequana/busco under Linux)

        from sequana import busco
        bd = busco.BuscoDownload()
        bd.download()

    Those datasets will then be found automatically by the BUSCO rule provided
    as a snakemake rule in Sequana.

    """

    def __init__(self):

        dv = DevTools()
        self.base = sequana_config_path + os.sep + "busco"
        dv.mkdir(self.base)
        self.filenames = sorted([
            "bacteria_odb9",
            "proteobacteria_odb9",
            "rhizobiales_odb9",
            "betaproteobacteria_odb9",
            "gammaproteobacteria_odb9",
            "enterobacteriales_odb9",
            "deltaepsilonsub_odb9",
            "actinobacteria_odb9",
            "cyanobacteria_odb9",
            "firmicutes_odb9",
            "clostridia_odb9",
            "lactobacillales_odb9",
            "bacillales_odb9",
            "bacteroidetes_odb9",
            "spirochaetes_odb9",
            "tenericutes_odb9",
            "eukaryota_odb9",
            "fungi_odb9",
            "microsporidia_odb9",
            "dikarya_odb9",
            "ascomycota_odb9",
            "pezizomycotina_odb9",
            "eurotiomycetes_odb9",
            "sordariomyceta_odb9",
            "saccharomyceta_odb9",
            "saccharomycetales_odb9",
            "basidiomycota_odb9",
            "metazoa_odb9",
            "nematoda_odb9",
            "arthropoda_odb9",
            "insecta_odb9",
            "endopterygota_odb9",
            "hymenoptera_odb9",
            "diptera_odb9",
            "vertebrata_odb9",
            "actinopterygii_odb9",
            "tetrapoda_odb9",
            "aves_odb9",
            "mammalia_odb9",
            "euarchontoglires_odb9",
            "laurasiatheria_odb9",
            "embryophyta_odb9",
            "protists_ensembl",
            "alveolata_stramenophiles_ensembl"])

    def download(self, uncompress=True):
        """Download the datasets (tar.gz) and uncompress them

        :param bool uncompress: if True, uncompress the tar.gz and delete it
        """
        url = "http://busco.ezlab.org/v2/datasets"
        for filename in self.filenames:
            basename = filename + ".tar.gz"
            target = self.base + "/" + basename
            print(url + "/" + basename)
            wget(url + "/" + basename, target)
            # TODO untar datasets and cleanup the tar.gz
            if uncompress:
                execute("tar xvfz %s -C %s" % (target, self.base))
                execute("rm -f %s" % ( target))

<commit_msg>add busco code<commit_after>import os

from sequana.misc import wget
from sequana import sequana_config_path
from easydev import DevTools, execute

class BuscoConfig(object):

    def __init__(self, species, outpath=None, sample_name=None, conda_bin_path=None,
            tmp_path=None, Rscript_bin_path=None):

        self.params = {}
        self.params['conda_bin_path'] = conda_bin_path # "/home/cokelaer/miniconda3/env/py3/bin"
        self.params['Rscript'] = Rscript_bin_path # "/home/cokelaer/miniconda3/bin"
        self.params['sample'] = sample_name  # prefix used in output files
        self.params['outpath'] = outpath  # where to store the output directory
        self.params['lineage_path'] = sequana_config_path + "/busco/{}".format(species)
        self.params['tmp_path'] = tmp_path

    def save_config_file(self, filename):
        from sequana import sequana_data
        config_generic = sequana_data("config.ini", "busco")
        data = open(config_generic, "r").read()
        data = data.format(**self.params)
        with open(filename, "w") as fh:
            fh.write(data)

class BuscoDownload(object):
    """

    This code downloads the BUSCO datasets (about G uncompressed). The data
    is stored in your config path (e.g. ~/.config/sequana/busco under Linux)

        from sequana import busco
        bd = busco.BuscoDownload()
        bd.download()

    Those datasets will then be found automatically by the BUSCO rule provided
    as a snakemake rule in Sequana.

    """

    def __init__(self):

        dv = DevTools()
        self.base = sequana_config_path + os.sep + "busco"
        dv.mkdir(self.base)
        self.filenames = sorted([
            "bacteria_odb9",
            "proteobacteria_odb9",
            "rhizobiales_odb9",
            "betaproteobacteria_odb9",
            "gammaproteobacteria_odb9",
            "enterobacteriales_odb9",
            "deltaepsilonsub_odb9",
            "actinobacteria_odb9",
            "cyanobacteria_odb9",
            "firmicutes_odb9",
            "clostridia_odb9",
            "lactobacillales_odb9",
            "bacillales_odb9",
            "bacteroidetes_odb9",
            "spirochaetes_odb9",
            "tenericutes_odb9",
            "eukaryota_odb9",
            "fungi_odb9",
            "microsporidia_odb9",
            "dikarya_odb9",
            "ascomycota_odb9",
            "pezizomycotina_odb9",
            "eurotiomycetes_odb9",
            "sordariomyceta_odb9",
            "saccharomyceta_odb9",
            "saccharomycetales_odb9",
            "basidiomycota_odb9",
            "metazoa_odb9",
            "nematoda_odb9",
            "arthropoda_odb9",
            "insecta_odb9",
            "endopterygota_odb9",
            "hymenoptera_odb9",
            "diptera_odb9",
            "vertebrata_odb9",
            "actinopterygii_odb9",
            "tetrapoda_odb9",
            "aves_odb9",
            "mammalia_odb9",
            "euarchontoglires_odb9",
            "laurasiatheria_odb9",
            "embryophyta_odb9",
            "protists_ensembl",
            "alveolata_stramenophiles_ensembl"])

    def download(self, uncompress=True):
        """Download the datasets (tar.gz) and uncompress them

        :param bool uncompress: if True, uncompress the tar.gz and delete it
        """
        url = "http://busco.ezlab.org/v2/datasets"
        for filename in self.filenames:
            basename = filename + ".tar.gz"
            target = self.base + "/" + basename
            print(url + "/" + basename)
            wget(url + "/" + basename, target)
            # TODO untar datasets and cleanup the tar.gz
            if uncompress:
                execute("tar xvfz %s -C %s" % (target, self.base))
                execute("rm -f %s" % ( target))

class BuscoAnalysis(object):

    def __init__(self, bin_path=None, config_path=None, species=None,
                sample_name=None, outpath=None):
        """

        :param bin_path: path to busco binary file.
        :param config_path: is the path to augustus config path. See example
            in sequana/resources/busco
        :param species: one of busco dataset (e.g. bacteria_odb9)
        :param sample_name: prefix used for all files
        :param outpath: main output path

        mode    = config['busco']['mode_choice'],
        options = config['busco']['options'],
        wkdir   =  __busco__workdir,
        species = config['busco']['species_choice'],

        If bin_path or config_path are none, uses CONDA_PREFIX or
        CONDA_ENV_PATH.

        """
        assert outpath is not None
        assert sample_name is not None
        assert species is not None

        import os
        #sample_name = params.wkdir.split(os.sep)[0]
        if bin_path is None:
            # try conda
            try:
                self.conda_bin_path = os.environ['CONDA_PREFIX'] + os.sep +"bin"
            except:
                self.conda_bin_path = os.environ['CONDA_ENV_PATH'] + os.sep +"bin"
        else:
            self.conda_bin_path = bin_path

        if config_path is None:
            try:
                self.augustus_config_path = os.environ['CONDA_PREFIX'] + os.sep + "config"
            except KeyError:
                self.augustus_config_path = os.environ['CONDA_ENV_PATH'] + os.sep + "config"
        else:
            self.augustus_config_path = config_path

        self.species = species # TODO make it an attribute that calls set_config
        self.sample_name = sample_name
        self.outpath = outpath

    def _set_config(self):
        config = BuscoConfig(
            self.species,
            sample_name=self.sample_name,
            outpath=self.outpath,
            conda_bin_path=self.conda_bin_path,
            Rscript_bin_path="", # not required by our analysis
            tmp_path="./tmp_{}".format(self.sample_name)
        )
        from easydev import mkdirs
        mkdirs(self.outpath)
        self.config_filename = self.outpath + "/config.ini"
        config.save_config_file(self.config_filename)

    def run(self, infile, threads=4, mode="genome"):
        """
        mode_choice: genome, transcriptomics, proteins
        species: name of a BUSCO dataset. (use Sequanix to get the list)
        options: any options understood by busco
        """
        self._set_config()
        # -f to force overwritten existing files
        cmd = "export BUSCO_CONFIG_FILE={};".format(self.config_filename)
        cmd += "export AUGUSTUS_CONFIG_PATH={};".format(self.augustus_config_path)
        cmd += "run_busco -i {infile} -m {mode} -c {threads} -f 2>>{log};"
        cmd = cmd.format(**{"infile":infile, 'mode':mode, "threads":threads, "log":"log"})

        # Note that execute() does not keep the shell info
        from easydev import execute as shell
        from snakemake import shell
        shell(cmd)
        shell("rm -rf ./tmp_{}".format(self.sample_name))

    def multirun(self, infile, threads=4, mode="genome", species=[]):
        """

        datasets = BuscoDownload().filenames
        res = b.multirun("canu_test10X.contigs.fasta", threads=1,
            species=datasets)

        """
        res = {}
        buffer_ = self.species
        for this in species:
            self.species = this
            try:
                self.run(infile, threads=threads, mode=mode)
                from sequana.assembly import BUSCO
                ab = BUSCO("{}/run_{}/full_table_{}.tsv".format(
                    self.outpath, self.sample_name, self.sample_name))
                res[this] = ab.score
            except:
                pass
        self.species = buffer_

        return res
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This is an example script to upload data (images, text, metadata) to
# google cloud storage and datastore (for general data)

# Preparation of Pubmed Open Access Data

ftp_base = "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc"
file_list = "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_file_list.txt"

from glob import glob
import requests
import signal
import xmltodict
import imghdr
import tempfile
import shutil
import tarfile
import numpy
import urllib
import pandas
import os
import re
import pwd

######################################################################
# Pubmed Functions
######################################################################

def get_userhome():
    '''get userhome gets the user's home based on the uid,
    and NOT the environment variable which is not consistent'''
    return pwd.getpwuid(os.getuid())[5]

def get_pubmed(download_base=None):
    '''retrieve pubmed database, either from locally saved file,
    or if not yet generated, obtain from FTP server
    '''
    if download_base is None:
        download_base = get_userhome()
    output_file = '%s/pmc.tsv' %(download_base)
    if not os.path.exists(output_file):
        download_folder = tempfile.mkdtemp()
        pmc_file = '%s/pmc.txt' %download_folder
        urllib.request.urlretrieve(file_list, pmc_file)
        pmc = pandas.read_csv(pmc_file,sep="\t",skiprows=1,header=None)
        pmc.columns = ["TARGZ_FILE","JOURNAL","PMCID","PMID","LICENSE"]
        pmc.to_csv(output_file,sep="\t",index=None)
        return pmc
    return pandas.read_csv(output_file,sep="\t")

def read_file(file_path):
    with open (file_path, "r") as myfile:
        return myfile.read().replace('\n', '')

def read_xml(xml_file):
    with open(xml_file) as fd:
        return xmltodict.parse(fd.read())

def format_name(name):
    '''format name will ensure that all collection names have
    periods removed, lowercase, and spaces replaced with -
    I'm not sure if this is best, but it's safer than allowing anything
    '''
    return name.replace('.','').lower().replace(" ",'-')

def get_metadata(row):
    '''get_uid will return the metadata for a row, with the uid corresponding
    first to the PMID, and the PMC id if that is not defined
    '''
    pmid = row[1].PMID
    if not isinstance(pmid,str):
        if numpy.isnan(pmid):
            pmid = row[1].PMCID
    download_url = "%s/%s" %(ftp_base,row[1].TARGZ_FILE)
    metadata = {"pmcid":row[1].PMCID,
                "type":"article",
                "uid":pmid,
                "publication_date": publication_date,
                "download_url":download_url,
                "license":row[1].LICENSE}
    if not isinstance(row[1].PMID,str):
        if not numpy.isnan(row[1].PMID):
            metadata['pmid'] = row[1].PMID
    return metadata

def create_article(metadata):
    tmpdir = tempfile.mkdtemp()
    pmc_file = '%s/article.tar.gz' %(tmpdir)
    print('Downloading: %s' %(metadata['uid']))
    urllib.request.urlretrieve(metadata['download_url'], pmc_file)
    tar = tarfile.open(pmc_file, "r:gz")
    tar.extractall(tmpdir)
    files = glob('%s/%s/*' %(tmpdir,metadata['pmcid']))
    images = [x for x in files if imghdr.what(x) is not None]
    pdf_files = [x for x in files if x.lower().endswith('pdf')]
    xml_file = [x for x in files if x.lower().endswith('xml')]
    images = images + pdf_files
    general_client.upload_dataset(images=images,
                                  texts=xml_file,
                                  collection=collection,
                                  uid=metadata['uid'],
                                  metadata=metadata)
    shutil.rmtree(tmpdir)

######################################################################
# Signals
######################################################################

def signal_handler(signum, frame):
    raise Exception("Timed out!")

# Only allow each paper a 30 seconds to download
signal.signal(signal.SIGALRM, signal_handler)

######################################################################
# Preparation of Pubmed Open Access Data
######################################################################

import os
import tempfile
import tarfile
import imghdr
import urllib
from glob import glob

# Obtain 1.5 million pmc articles
pmc = get_pubmed()

# Start google storage client for pmc-stanford
from som.api.google.storage.general import Client
general_client = Client(bucket_name='pmc-stanford')

for row in pmc.iterrows():
        journal_name = row[1].JOURNAL
        date_match = re.search("\d{4}",journal_name)
        publication_date = journal_name[date_match.start():]
        journal_name = format_name(journal_name[:date_match.start()].strip())
        collection = general_client.get_collection(fields={"uid":journal_name})
        metadata = get_metadata(row)
        article = create_article(metadata)

<commit_msg>	modified:   examples/general/upload_storage.py<commit_after>#!/usr/bin/env python

# This is an example script to upload data (images, text, metadata) to
# google cloud storage and datastore (for general data)

# Preparation of Pubmed Open Access Data

ftp_base = "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc"
file_list = "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_file_list.txt"

from glob import glob
import requests
import signal
import xmltodict
import imghdr
import tempfile
import shutil
import tarfile
import numpy
import urllib
import pandas
import os
import re
import pwd

######################################################################
# Pubmed Functions
######################################################################

def get_userhome():
    '''get userhome gets the user's home based on the uid,
    and NOT the environment variable which is not consistent'''
    return pwd.getpwuid(os.getuid())[5]

def get_pubmed(download_base=None):
    '''retrieve pubmed database, either from locally saved file,
    or if not yet generated, obtain from FTP server
    '''
    if download_base is None:
        download_base = get_userhome()
    output_file = '%s/pmc.tsv' %(download_base)
    if not os.path.exists(output_file):
        download_folder = tempfile.mkdtemp()
        pmc_file = '%s/pmc.txt' %download_folder
        urllib.request.urlretrieve(file_list, pmc_file)
        pmc = pandas.read_csv(pmc_file,sep="\t",skiprows=1,header=None)
        pmc.columns = ["TARGZ_FILE","JOURNAL","PMCID","PMID","LICENSE"]
        pmc.to_csv(output_file,sep="\t",index=None)
        return pmc
    return pandas.read_csv(output_file,sep="\t")

def read_file(file_path):
    with open (file_path, "r") as myfile:
        return myfile.read().replace('\n', '')

def read_xml(xml_file):
    with open(xml_file) as fd:
        return xmltodict.parse(fd.read())

def format_name(name):
    '''format name will ensure that all collection names have
    periods removed, lowercase, and spaces replaced with -
    I'm not sure if this is best, but it's safer than allowing anything
    '''
    return name.replace('.','').lower().replace(" ",'-')

def get_metadata(row):
    '''get_uid will return the metadata for a row, with the uid corresponding
    first to the PMID, and the PMC id if that is not defined
    '''
    pmid = row[1].PMID
    if not isinstance(pmid,str):
        if numpy.isnan(pmid):
            pmid = row[1].PMCID
    download_url = "%s/%s" %(ftp_base,row[1].TARGZ_FILE)
    metadata = {"pmcid":row[1].PMCID,
                "type":"article",
                "uid":pmid,
                "publication_date": publication_date,
                "download_url":download_url,
                "license":row[1].LICENSE}
    if not isinstance(row[1].PMID,str):
        if not numpy.isnan(row[1].PMID):
            metadata['pmid'] = row[1].PMID
    return metadata

def create_article(metadata):
    tmpdir = tempfile.mkdtemp()
    pmc_file = '%s/article.tar.gz' %(tmpdir)
    print('Downloading: %s' %(metadata['uid']))
    urllib.request.urlretrieve(metadata['download_url'], pmc_file)
    tar = tarfile.open(pmc_file, "r:gz")
    tar.extractall(tmpdir)
    files = glob('%s/%s/*' %(tmpdir,metadata['pmcid']))
    images = [x for x in files if imghdr.what(x) is not None]
    pdf_files = [x for x in files if x.lower().endswith('pdf')]
    xml_file = [x for x in files if x.lower().endswith('xml')]
    images = images + pdf_files
    general_client.upload_dataset(images=images,
                                  texts=xml_file,
                                  collection=collection,
                                  uid=metadata['uid'],
                                  metadata=metadata)
    shutil.rmtree(tmpdir)

######################################################################
# Signals
######################################################################

def signal_handler(signum, frame):
    raise Exception("Timed out!")

# Only allow each paper a 30 seconds to download
signal.signal(signal.SIGALRM, signal_handler)

######################################################################
# Preparation of Pubmed Open Access Data
######################################################################

import os
import tempfile
import tarfile
import imghdr
import urllib
from glob import glob

# Obtain 1.5 million pmc articles
pmc = get_pubmed()

# Start google storage client for pmc-stanford
from som.api.google.storage.general import Client
general_client = Client(bucket_name='pmc-stanford')

for row in pmc.iterrows():
        journal_name = row[1].JOURNAL
        date_match = re.search("\d{4}",journal_name)
        publication_date = journal_name[date_match.start():]
        journal_name = format_name(journal_name[:date_match.start()].strip())
        collection = general_client.get_collection({"uid":journal_name})
        metadata = get_metadata(row)
        article = create_article(metadata)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This example adds a hotel campaign, ad group, and ad group ad.

Prerequisite: You need to have access to the Hotel Ads Center, which can be
granted during integration with Google Hotels. The integration instructions can
be found at:
https://support.google.com/hotelprices/answer/6101897.
"""

import argparse
import sys
import uuid

import google.ads.google_ads.client

_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT = 20000000

def main(
    client, customer_id, hotel_center_account_id, bid_ceiling_micro_amount
):

    budget_resource_name = add_budget(client, customer_id)

    campaign_resource_name = add_hotel_campaign(
        client,
        customer_id,
        budget_resource_name,
        hotel_center_account_id,
        bid_ceiling_micro_amount,
    )

    ad_group_resource_name = add_hotel_ad_group(
        client, customer_id, campaign_resource_name
    )

    add_hotel_ad(client, customer_id, ad_group_resource_name)

def add_budget(client, customer_id):
    campaign_budget_service = client.get_service(
        "CampaignBudgetService", version="v6"
    )

    # Create a budget, which can be shared by multiple campaigns.
    campaign_budget_operation = client.get_type(
        "CampaignBudgetOperation", version="v6"
    )
    campaign_budget = campaign_budget_operation.create
    campaign_budget.name = "Interplanetary Budget %s" % uuid.uuid4()
    campaign_budget.delivery_method = client.get_type(
        "BudgetDeliveryMethodEnum"
    ).STANDARD
    campaign_budget.amount_micros = 500000

    # Add budget.
    try:
        campaign_budget_response = campaign_budget_service.mutate_campaign_budgets(
            customer_id, [campaign_budget_operation]
        )
    except google.ads.google_ads.errors.GoogleAdsException as ex:
        print(
            'Request with ID "%s" failed with status "%s" and includes the '
            "following errors:" % (ex.request_id, ex.error.code().name)
        )
        for error in ex.failure.errors:
            print('\tError with message "%s".' % error.message)
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print("\t\tOn field: %s" % field_path_element.field_name)
        sys.exit(1)

    budget_resource_name = campaign_budget_response.results[0].resource_name

    print("Created budget %s" % budget_resource_name)

    return budget_resource_name

# [START add_hotel_ad_1]
def add_hotel_ad(client, customer_id, ad_group_resource_name):
    ad_group_ad_service = client.get_service("AdGroupAdService", version="v6")

    # Creates a new ad group ad and sets the hotel ad to it.
    ad_group_ad_operation = client.get_type("AdGroupAdOperation", version="v6")
    ad_group_ad = ad_group_ad_operation.create
    ad_group_ad.ad_group = ad_group_resource_name
    # Set the ad group ad to enabled.  Setting this to paused will cause an error
    # for hotel campaigns.  For hotels pausing should happen at either the ad group or
    # campaign level.
    ad_group_ad.status = client.get_type(
        "AdGroupAdStatusEnum", version="v6"
    ).ENABLED
    ad_group_ad.ad.hotel_ad.CopyFrom(
        client.get_type("HotelAdInfo", version="v6")
    )

    # Add the ad group ad.
    try:
        ad_group_ad_response = ad_group_ad_service.mutate_ad_group_ads(
            customer_id, [ad_group_ad_operation]
        )
    except google.ads.google_ads.errors.GoogleAdsException as ex:
        print(
            'Request with ID "%s" failed with status "%s" and includes the '
            "following errors:" % (ex.request_id, ex.error.code().name)
        )
        for error in ex.failure.errors:
            print('\tError with message "%s".' % error.message)
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print("\t\tOn field: %s" % field_path_element.field_name)
        sys.exit(1)

    ad_group_ad_resource_name = ad_group_ad_response.results[0].resource_name

    print("Created hotel ad %s." % ad_group_ad_resource_name)

    return ad_group_resource_name
    # [END add_hotel_ad_1]

# [START add_hotel_ad_2]
def add_hotel_ad_group(client, customer_id, campaign_resource_name):
    ad_group_service = client.get_service("AdGroupService", version="v6")

    # Create ad group.
    ad_group_operation = client.get_type("AdGroupOperation", version="v6")
    ad_group = ad_group_operation.create
    ad_group.name = "Earth to Mars cruise %s" % uuid.uuid4()
    ad_group.status = client.get_type("AdGroupStatusEnum", version="v6").ENABLED
    ad_group.campaign = campaign_resource_name
    # Sets the ad group type to HOTEL_ADS. This cannot be set to other types.
    ad_group.type = client.get_type("AdGroupTypeEnum", version="v6").HOTEL_ADS
    ad_group.cpc_bid_micros = 10000000

    # Add the ad group.
    try:
        ad_group_response = ad_group_service.mutate_ad_groups(
            customer_id, [ad_group_operation]
        )
    except google.ads.google_ads.errors.GoogleAdsException as ex:
        print(
            'Request with ID "%s" failed with status "%s" and includes the '
            "following errors:" % (ex.request_id, ex.error.code().name)
        )
        for error in ex.failure.errors:
            print('\tError with message "%s".' % error.message)
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print("\t\tOn field: %s" % field_path_element.field_name)
        sys.exit(1)

    ad_group_resource_name = ad_group_response.results[0].resource_name

    print(
        'Added a hotel ad group with resource name "%s".'
        % ad_group_resource_name
    )

    return ad_group_resource_name
    # [END add_hotel_ad_2]

# [START add_hotel_ad_3]
def add_hotel_campaign(
    client,
    customer_id,
    budget_resource_name,
    hotel_center_account_id,
    bid_ceiling_micro_amount,
):
    campaign_service = client.get_service("CampaignService", version="v6")

    # [START add_hotel_ad]
    # Create campaign.
    campaign_operation = client.get_type("CampaignOperation", version="v6")
    campaign = campaign_operation.create
    campaign.name = "Interplanetary Cruise Campaign %s" % uuid.uuid4()

    # Configures settings related to hotel campaigns including advertising
    # channel type and hotel setting info.
    campaign.advertising_channel_type = client.get_type(
        "AdvertisingChannelTypeEnum"
    ).HOTEL
    campaign.hotel_setting.hotel_center_id = hotel_center_account_id

    # Recommendation: Set the campaign to PAUSED when creating it to prevent the
    # ads from immediately serving. Set to ENABLED once you've added targeting
    # and the ads are ready to serve.
    campaign.status = client.get_type("CampaignStatusEnum", version="v6").PAUSED

    # Set the bidding strategy to PercentCpc. Only Manual CPC and Percent CPC
    # can be used for hotel campaigns.
    campaign.percent_cpc.cpc_bid_ceiling_micros = bid_ceiling_micro_amount

    # Sets the budget.
    campaign.campaign_budget = budget_resource_name

    # Set the campaign network options. Only Google Search is allowed for hotel
    # campaigns.
    campaign.network_settings.target_google_search = True
    # [END add_hotel_ad]

    # Add the campaign.
    try:
        campaign_response = campaign_service.mutate_campaigns(
            customer_id, [campaign_operation]
        )
    except google.ads.google_ads.errors.GoogleAdsException as ex:
        print(
            'Request with ID "%s" failed with status "%s" and includes the '
            "following errors:" % (ex.request_id, ex.error.code().name)
        )
        for error in ex.failure.errors:
            print('\tError with message "%s".' % error.message)
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print("\t\tOn field: %s" % field_path_element.field_name)
        sys.exit(1)

    campaign_resource_name = campaign_response.results[0].resource_name

    print(
        'Added a hotel campaign with resource name "%s".'
        % campaign_resource_name
    )

    return campaign_resource_name
    # [END add_hotel_ad_3]

if __name__ == "__main__":
    # GoogleAdsClient will read the google-ads.yaml configuration file in the
    # home directory if none is specified.
    google_ads_client = (
        google.ads.google_ads.client.GoogleAdsClient.load_from_storage()
    )

    parser = argparse.ArgumentParser(
        description=(
            "Adds an expanded text ad to the specified ad group ID, "
            "for the given customer ID."
        )
    )
    # The following argument(s) should be provided to run the example.
    parser.add_argument(
        "-c",
        "--customer_id",
        type=str,
        required=True,
        help="The Google Ads customer ID.",
    )
    parser.add_argument(
        "-b",
        "--bid_ceiling_micro_amount",
        type=int,
        required=False,
        default=_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT,
        help=(
            "The bid ceiling micro amount for the hotel campaign. "
            f"The default value is {_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT}"
        ),
    )
    parser.add_argument(
        "-a",
        "--hotel_center_account_id",
        type=int,
        required=True,
        help="The hotel center account ID.",
    )
    args = parser.parse_args()

    main(
        google_ads_client,
        args.customer_id,
        args.hotel_center_account_id,
        args.bid_ceiling_micro_amount,
    )
<commit_msg>Fix param name in add_hotel_ads example (#359)<commit_after>#!/usr/bin/env python
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This example adds a hotel campaign, ad group, and ad group ad.

Prerequisite: You need to have access to the Hotel Ads Center, which can be
granted during integration with Google Hotels. The integration instructions can
be found at:
https://support.google.com/hotelprices/answer/6101897.
"""

import argparse
import sys
import uuid

from google.ads.google_ads.client import GoogleAdsClient
from google.ads.google_ads.errors import GoogleAdsException

_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT = 20000000

def main(
    client, customer_id, hotel_center_account_id, cpc_bid_ceiling_micro_amount
):

    try:
        budget_resource_name = add_budget(client, customer_id)

        campaign_resource_name = add_hotel_campaign(
            client,
            customer_id,
            budget_resource_name,
            hotel_center_account_id,
            cpc_bid_ceiling_micro_amount,
        )

        ad_group_resource_name = add_hotel_ad_group(
            client, customer_id, campaign_resource_name
        )

        add_hotel_ad(client, customer_id, ad_group_resource_name)
    except GoogleAdsException as ex:
        print(
            f'Request with ID "{ex.request_id}" failed with status '
            f'"{ex.error.code().name}" and includes the following errors:'
        )
        for error in ex.failure.errors:
            print(f'\tError with message "{error.message}".')
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print(f"\t\tOn field: {field_path_element.field_name}")
        sys.exit(1)

def add_budget(client, customer_id):
    campaign_budget_service = client.get_service(
        "CampaignBudgetService", version="v6"
    )

    # Create a budget, which can be shared by multiple campaigns.
    campaign_budget_operation = client.get_type(
        "CampaignBudgetOperation", version="v6"
    )
    campaign_budget = campaign_budget_operation.create
    campaign_budget.name = f"Interplanetary Budget #{uuid.uuid4()}"
    campaign_budget.delivery_method = client.get_type(
        "BudgetDeliveryMethodEnum"
    ).STANDARD
    campaign_budget.amount_micros = 500000

    # Add budget.
    campaign_budget_response = campaign_budget_service.mutate_campaign_budgets(
        customer_id, [campaign_budget_operation]
    )
    budget_resource_name = campaign_budget_response.results[0].resource_name

    print(f"Created budget with resource name: '{budget_resource_name}'")

    return budget_resource_name

# [START add_hotel_ad_1]
def add_hotel_ad(client, customer_id, ad_group_resource_name):
    ad_group_ad_service = client.get_service("AdGroupAdService", version="v6")

    # Creates a new ad group ad and sets the hotel ad to it.
    ad_group_ad_operation = client.get_type("AdGroupAdOperation", version="v6")
    ad_group_ad = ad_group_ad_operation.create
    ad_group_ad.ad_group = ad_group_resource_name
    # Set the ad group ad to enabled.  Setting this to paused will cause an error
    # for hotel campaigns.  For hotels pausing should happen at either the ad group or
    # campaign level.
    ad_group_ad.status = client.get_type(
        "AdGroupAdStatusEnum", version="v6"
    ).ENABLED
    ad_group_ad.ad.hotel_ad.CopyFrom(
        client.get_type("HotelAdInfo", version="v6")
    )

    # Add the ad group ad.
    ad_group_ad_response = ad_group_ad_service.mutate_ad_group_ads(
        customer_id, [ad_group_ad_operation]
    )
    ad_group_ad_resource_name = ad_group_ad_response.results[0].resource_name

    print(f"Created hotel ad with resource name: '{ad_group_ad_resource_name}'")

    return ad_group_resource_name
    # [END add_hotel_ad_1]

# [START add_hotel_ad_2]
def add_hotel_ad_group(client, customer_id, campaign_resource_name):
    ad_group_service = client.get_service("AdGroupService", version="v6")

    # Create ad group.
    ad_group_operation = client.get_type("AdGroupOperation", version="v6")
    ad_group = ad_group_operation.create
    ad_group.name = f"Earth to Mars cruise #{uuid.uuid4()}"
    ad_group.status = client.get_type("AdGroupStatusEnum", version="v6").ENABLED
    ad_group.campaign = campaign_resource_name
    # Sets the ad group type to HOTEL_ADS. This cannot be set to other types.
    ad_group.type = client.get_type("AdGroupTypeEnum", version="v6").HOTEL_ADS
    ad_group.cpc_bid_micros = 10000000

    # Add the ad group.
    ad_group_response = ad_group_service.mutate_ad_groups(
        customer_id, [ad_group_operation]
    )
    ad_group_resource_name = ad_group_response.results[0].resource_name

    print(
        f"Added hotel ad group with resource name: '{ad_group_resource_name}'"
    )

    return ad_group_resource_name
    # [END add_hotel_ad_2]

# [START add_hotel_ad_3]
def add_hotel_campaign(
    client,
    customer_id,
    budget_resource_name,
    hotel_center_account_id,
    cpc_bid_ceiling_micro_amount,
):
    campaign_service = client.get_service("CampaignService", version="v6")

    # [START add_hotel_ad]
    # Create campaign.
    campaign_operation = client.get_type("CampaignOperation", version="v6")
    campaign = campaign_operation.create
    campaign.name = f"Interplanetary Cruise Campaign #{uuid.uuid4()}"

    # Configures settings related to hotel campaigns including advertising
    # channel type and hotel setting info.
    campaign.advertising_channel_type = client.get_type(
        "AdvertisingChannelTypeEnum"
    ).HOTEL
    campaign.hotel_setting.hotel_center_id = hotel_center_account_id

    # Recommendation: Set the campaign to PAUSED when creating it to prevent the
    # ads from immediately serving. Set to ENABLED once you've added targeting
    # and the ads are ready to serve.
    campaign.status = client.get_type("CampaignStatusEnum", version="v6").PAUSED

    # Set the bidding strategy to PercentCpc. Only Manual CPC and Percent CPC
    # can be used for hotel campaigns.
    campaign.percent_cpc.cpc_bid_ceiling_micros = cpc_bid_ceiling_micro_amount

    # Sets the budget.
    campaign.campaign_budget = budget_resource_name

    # Set the campaign network options. Only Google Search is allowed for hotel
    # campaigns.
    campaign.network_settings.target_google_search = True
    # [END add_hotel_ad]

    # Add the campaign.
    campaign_response = campaign_service.mutate_campaigns(
        customer_id, [campaign_operation]
    )
    campaign_resource_name = campaign_response.results[0].resource_name

    print(
        f"Added a hotel campaign with resource name: '{campaign_resource_name}'"
    )

    return campaign_resource_name
    # [END add_hotel_ad_3]

if __name__ == "__main__":
    # GoogleAdsClient will read the google-ads.yaml configuration file in the
    # home directory if none is specified.
    google_ads_client = GoogleAdsClient.load_from_storage()

    parser = argparse.ArgumentParser(
        description=(
            "Adds an expanded text ad to the specified ad group ID, "
            "for the given customer ID."
        )
    )
    # The following argument(s) should be provided to run the example.
    parser.add_argument(
        "-c",
        "--customer_id",
        type=str,
        required=True,
        help="The Google Ads customer ID.",
    )
    parser.add_argument(
        "-b",
        "--cpc_bid_ceiling_micro_amount",
        type=int,
        required=False,
        default=_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT,
        help=(
            "The bid ceiling micro amount for the hotel campaign. "
            f"The default value is {_DEFAULT_CPC_BID_CEILING_MICRO_AMOUNT}"
        ),
    )
    parser.add_argument(
        "-a",
        "--hotel_center_account_id",
        type=int,
        required=True,
        help="The hotel center account ID.",
    )
    args = parser.parse_args()

    main(
        google_ads_client,
        args.customer_id,
        args.hotel_center_account_id,
        args.cpc_bid_ceiling_micro_amount,
    )
<|endoftext|>


<commit_before>import csv
from urllib.parse import urljoin
from datetime import datetime
import unicodedata, re

fieldnames = ['name',
 'title',
 'url',
 'author',
 'publisher',
 'issued',
 'publisher_classification',
 'description',
 'tags',
 'license_id',
 'license_url',
 'place',
 'location',
 'country',
 'language',
 'status',
 'metadatacreated',
 'generator',
 'api_endpoint',
 'api_type',
 'full_metadata_download']

def slug(name):
    'Creates a slug identifier from a name'
    slug = unicodedata.normalize('NFKD', name)
    slug = slug.lower()
    slug = re.sub(' ','-',slug)
    slug = re.sub('[^a-z0-9-]+', '', slug).strip('-')
    slug = re.sub('[-]+', '-', slug)
    return slug

# Read the data from the Brazilian repository of data catalogs

with open('../dados/catalogos.csv') as br_csv:
    br_catalogs = csv.DictReader(br_csv)
    with open ('portals.csv', 'w') as portals_csv:
        br_dataportals = csv.DictWriter(portals_csv, fieldnames=fieldnames)
        br_dataportals.writeheader()
        for br_catalog in br_catalogs:
            # Prepare a blank line
            new_portal = dict(map(lambda key: (key, ''),fieldnames))
            # Map values
            new_portal['name'] = slug(br_catalog.get('Título',''))
            new_portal['title'] = br_catalog.get('Título','')
            new_portal['url'] = br_catalog['URL']
            new_portal['country'] = 'BR'
            new_portal['language'] = 'pt'
            new_portal['status'] = 'active'
            new_portal['publisher_classification'] = 'Government'
            tags = []
            # Is it a city catalog?
            if br_catalog.get('Município','') and br_catalog.get('UF',''):
                new_portal['place'] = ', '.join((br_catalog['Município'], br_catalog['UF'], 'Brazil'))
                if br_catalog.get('Poder','') == 'Executivo':
                    new_portal['description'] = 'Open Data portal of the city of {}, {}, in Brazil'.format(br_catalog['Município'], br_catalog['UF'])
                elif br_catalog['Poder'] == 'Legislativo':
                    new_portal['description'] = 'Open Data portal of the local parliament of {}, {}, in Brazil'.format(br_catalog['Município'], br_catalog['UF'])
                tags.append('city')
                tags.append('local-government')
                tags.append('br-' + br_catalog['UF'])
                tags.append(br_catalog['Município'])
            # or is it a state catalog?
            elif br_catalog.get('UF',''):
                new_portal['place'] = ', '.join((br_catalog['UF'], 'Brazil'))
                if br_catalog.get('Poder','') == 'Executivo':
                    new_portal['description'] = 'Open Data portal of the state of {}, in Brazil'.format(br_catalog['UF'])
                elif br_catalog['Poder'] == 'Legislativo':
                    new_portal['description'] = 'Open Data portal of the local parliament of {}, in Brazil'.format(br_catalog['UF'])
                tags.append('local-government')
                tags.append('br-' + br_catalog['UF'])
            # Does it use CKAN?
            if br_catalog.get('Solução','') == 'CKAN':
                tags.append(br_catalog['Solução'])
                new_portal['generator'] = br_catalog['Solução']
                new_portal['api_endpoint'] = urljoin(br_catalog['URL'], '/api/3')
                new_portal['api_type'] = 'CKAN API'
            new_portal['tags'] = ','.join(tags)
            new_portal['metadatacreated'] = str(datetime.now())
            br_dataportals.writerow(new_portal)

<commit_msg>corrige o formato do timestamp<commit_after>import csv
from urllib.parse import urljoin
from datetime import datetime
import unicodedata, re

fieldnames = ['name',
 'title',
 'url',
 'author',
 'publisher',
 'issued',
 'publisher_classification',
 'description',
 'tags',
 'license_id',
 'license_url',
 'place',
 'location',
 'country',
 'language',
 'status',
 'metadatacreated',
 'generator',
 'api_endpoint',
 'api_type',
 'full_metadata_download']

def slug(name):
    'Creates a slug identifier from a name'
    slug = unicodedata.normalize('NFKD', name)
    slug = slug.lower()
    slug = re.sub(' ','-',slug)
    slug = re.sub('[^a-z0-9-]+', '', slug).strip('-')
    slug = re.sub('[-]+', '-', slug)
    return slug

# Read the data from the Brazilian repository of data catalogs

with open('../dados/catalogos.csv') as br_csv:
    br_catalogs = csv.DictReader(br_csv)
    with open ('portals.csv', 'w') as portals_csv:
        br_dataportals = csv.DictWriter(portals_csv, fieldnames=fieldnames)
        br_dataportals.writeheader()
        for br_catalog in br_catalogs:
            # Prepare a blank line
            new_portal = dict(map(lambda key: (key, ''),fieldnames))
            # Map values
            new_portal['name'] = slug(br_catalog.get('Título',''))
            new_portal['title'] = br_catalog.get('Título','')
            new_portal['url'] = br_catalog['URL']
            new_portal['country'] = 'BR'
            new_portal['language'] = 'pt'
            new_portal['status'] = 'active'
            new_portal['publisher_classification'] = 'Government'
            tags = []
            # Is it a city catalog?
            if br_catalog.get('Município','') and br_catalog.get('UF',''):
                new_portal['place'] = ', '.join((br_catalog['Município'], br_catalog['UF'], 'Brazil'))
                if br_catalog.get('Poder','') == 'Executivo':
                    new_portal['description'] = 'Open Data portal of the city of {}, {}, in Brazil'.format(br_catalog['Município'], br_catalog['UF'])
                elif br_catalog['Poder'] == 'Legislativo':
                    new_portal['description'] = 'Open Data portal of the local parliament of {}, {}, in Brazil'.format(br_catalog['Município'], br_catalog['UF'])
                tags.append('city')
                tags.append('local-government')
                tags.append('br-' + br_catalog['UF'])
                tags.append(br_catalog['Município'])
            # or is it a state catalog?
            elif br_catalog.get('UF',''):
                new_portal['place'] = ', '.join((br_catalog['UF'], 'Brazil'))
                if br_catalog.get('Poder','') == 'Executivo':
                    new_portal['description'] = 'Open Data portal of the state of {}, in Brazil'.format(br_catalog['UF'])
                elif br_catalog['Poder'] == 'Legislativo':
                    new_portal['description'] = 'Open Data portal of the local parliament of {}, in Brazil'.format(br_catalog['UF'])
                tags.append('local-government')
                tags.append('br-' + br_catalog['UF'])
            # Does it use CKAN?
            if br_catalog.get('Solução','') == 'CKAN':
                tags.append(br_catalog['Solução'])
                new_portal['generator'] = br_catalog['Solução']
                new_portal['api_endpoint'] = urljoin(br_catalog['URL'], '/api/3')
                new_portal['api_type'] = 'CKAN API'
            new_portal['tags'] = ','.join(tags)
            new_portal['metadatacreated'] = datetime.now().isoformat(timespec='seconds')+'Z'
            br_dataportals.writerow(new_portal)

<|endoftext|>