<commit_before># -*- coding: utf8 -*-
import json
import logging
import redis

import requests
from django.db import models
from django.conf import settings
from django.core.urlresolvers import reverse
from social_auth.db.django_models import UserSocialAuth

from frigg.helpers import github
from .managers import ProjectManager

logger = logging.getLogger("frigg_build_logger")

class Project(models.Model):
    name = models.CharField(max_length=100, blank=True)
    owner = models.CharField(max_length=100, blank=True)
    git_repository = models.CharField(max_length=150)
    average_time = models.IntegerField(null=True)
    private = models.BooleanField(default=True)
    approved = models.BooleanField(default=False)
    user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True,
                             help_text='A user with access to the repository.')

    objects = ProjectManager()

    def __unicode__(self):
        return "%(owner)s / %(name)s " % self.__dict__

    @property
    def github_token(self):
        try:
            token = UserSocialAuth.objects.get(user=self.user,
                                               provider='github').extra_data['access_token']
        except UserSocialAuth.DoesNotExist:
            token = getattr(settings, 'GITHUB_ACCESS_TOKEN', ':')
        return token

    @property
    def clone_url(self):
        if self.private:
            return "https://%s@github.com/%s/%s" % (self.github_token, self.owner, self.name)
        else:
            return "https://github.com/%s/%s" % (self.owner, self.name)

    @property
    def last_build_number(self):
        try:
            return self.builds.all().order_by('-build_number')[0].build_number
        except IndexError:
            return 0

    def start_build(self, data):
        return Build.objects.create(
            project=self,
            build_number=self.last_build_number + 1,
            pull_request_id=data['pull_request_id'],
            branch=data['branch'],
            sha=data["sha"]
        ).start()

    @classmethod
    def token_for_url(cls, repo_url):
        try:
            project = Project.objects.get(git_repository=repo_url)
            token = project.github_token
        except cls.DoesNotExist:
            token = getattr(settings, 'GITHUB_ACCESS_TOKEN', ':')
        return token

class Build(models.Model):
    project = models.ForeignKey(Project, related_name='builds', null=True)
    build_number = models.IntegerField(db_index=True)
    pull_request_id = models.IntegerField(max_length=150, default=0)
    branch = models.CharField(max_length=100, default="master")
    sha = models.CharField(max_length=150)

    class Meta:
        unique_together = ('project', 'build_number')

    def __unicode__(self):
        return "%s / %s " % (self.project, self.branch)

    def get_absolute_url(self):
        return "https://%s%s" % (
            settings.SERVER_ADDRESS,
            reverse('view_build', args=[self.project.owner, self.project.name, self.build_number])
        )

    def get_pull_request_url(self):
        return github.get_pull_request_url(self)

    @property
    def is_pending(self):
        return hasattr(self, 'result')

    @property
    def color(self):
        if self.is_pending:
            return 'orange'
        if self.result.succeeded:
            return 'green'
        return 'red'

    @property
    def comment_message(self):
        if self.succeeded:
            return "All gooodie good\n\n%s" % self.get_absolute_url()
        else:
            return "Be careful.. the tests failed\n\n%s" % self.get_absolute_url()

    @property
    def queue_object(self):
        return {
            'id': self.pk,
            'branch': self.branch,
            'sha': self.sha,
            'clone_url': self.project.clone_url,
            'owner': self.project.owner,
            'name': self.project.name
        }

    def start(self):
        if hasattr(self, 'result'):
            self.result.delete()

        if not self.approved:
            return BuildResult.create_not_approved(self)

        github.set_commit_status(self, pending=True)

        r = redis.Redis(**settings.REDIS_SETTINGS)
        r.lpush(settings.FRIGG_WORKER_QUEUE, json.dumps(self.queue_object))

        return self

    def handle_worker_report(self, payload):
        BuildResult.create_from_worker_payload(self, payload)

        github.set_commit_status(self)
        if 'comment' in payload and payload['comment']:
            github.comment_on_commit(self, self.comment_message)

        if 'webhooks' in payload:
            for url in payload['webhooks']:
                self.send_webhook(url)

    def send_webhook(self, url):
        return requests.post(url, data=json.dumps({
            'repository': self.project.git_repository,
            'sha': self.sha,
            'build_url': self.get_absolute_url(),
            'pull_request_id': self.pull_request_id,
            'state': self.result.succeeded,
            'return_code': self.result.return_code
        }))

class BuildResult(models.Model):
    build = models.OneToOneField(Build, related_name='result')
    result_log = models.TextField()
    succeeded = models.BooleanField(default=False)
    return_code = models.CharField(max_length=100)

    def __unicode__(self):
        return "%s - %s" % (self.build, self.build.build_number)

    @classmethod
    def create_not_approved(cls, build):
        cls.objects.create(build=build, result_log='This project is not approved.', succeeded=False)
        github.set_commit_status(build, error='This project is not approved')

    @classmethod
    def create_from_worker_payload(cls, build, payload):
        result = cls.objects.create(build_id=build.pk, succeeded=True, return_code='',
                                    result_log='')
        return_codes = []
        for r in payload['results']:
            if 'return_code' in r:
                return_codes.append(r['return_code'])
            result.succeeded = 'succeeded' in r and r['succeeded'] and result.succeeded
            result.result_log += cls.create_log_string_for_task(r)
        result.return_code = ",".join([str(code) for code in return_codes])
        result.save()

    @classmethod
    def create_log_string_for_task(cls, result):
        if 'task' in result and 'log' in result and 'return_code' in result:
            return ('Task: %(task)s\n'
                    '\n------------------------------------\n'
                    '%(log)s'
                    '\n------------------------------------\n'
                    'Exited with exit code: %(return_code)s\n\n') % result
        return ''
<commit_msg>Change __unicode__ to __str__<commit_after># -*- coding: utf8 -*-
import json
import logging
import redis

import requests
from django.db import models
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.encoding import python_2_unicode_compatible
from social_auth.db.django_models import UserSocialAuth

from frigg.helpers import github
from .managers import ProjectManager

logger = logging.getLogger(__name__)

@python_2_unicode_compatible
class Project(models.Model):
    name = models.CharField(max_length=100, blank=True)
    owner = models.CharField(max_length=100, blank=True)
    git_repository = models.CharField(max_length=150)
    average_time = models.IntegerField(null=True)
    private = models.BooleanField(default=True)
    approved = models.BooleanField(default=False)
    user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True,
                             help_text='A user with access to the repository.')

    objects = ProjectManager()

    def __str__(self):
        return "%(owner)s / %(name)s " % self.__dict__

    @property
    def github_token(self):
        try:
            token = UserSocialAuth.objects.get(user=self.user,
                                               provider='github').extra_data['access_token']
        except UserSocialAuth.DoesNotExist:
            token = getattr(settings, 'GITHUB_ACCESS_TOKEN', ':')
        return token

    @property
    def clone_url(self):
        if self.private:
            return "https://%s@github.com/%s/%s" % (self.github_token, self.owner, self.name)
        else:
            return "https://github.com/%s/%s" % (self.owner, self.name)

    @property
    def last_build_number(self):
        try:
            return self.builds.all().order_by('-build_number')[0].build_number
        except IndexError:
            return 0

    def start_build(self, data):
        return Build.objects.create(
            project=self,
            build_number=self.last_build_number + 1,
            pull_request_id=data['pull_request_id'],
            branch=data['branch'],
            sha=data["sha"]
        ).start()

    @classmethod
    def token_for_url(cls, repo_url):
        try:
            project = Project.objects.get(git_repository=repo_url)
            token = project.github_token
        except cls.DoesNotExist:
            token = getattr(settings, 'GITHUB_ACCESS_TOKEN', ':')
        return token

@python_2_unicode_compatible
class Build(models.Model):
    project = models.ForeignKey(Project, related_name='builds', null=True)
    build_number = models.IntegerField(db_index=True)
    pull_request_id = models.IntegerField(max_length=150, default=0)
    branch = models.CharField(max_length=100, default="master")
    sha = models.CharField(max_length=150)

    class Meta:
        unique_together = ('project', 'build_number')

    def __str__(self):
        return "%s / %s " % (self.project, self.branch)

    def get_absolute_url(self):
        return "https://%s%s" % (
            settings.SERVER_ADDRESS,
            reverse('view_build', args=[self.project.owner, self.project.name, self.build_number])
        )

    def get_pull_request_url(self):
        return github.get_pull_request_url(self)

    @property
    def is_pending(self):
        return hasattr(self, 'result')

    @property
    def color(self):
        if self.is_pending:
            return 'orange'
        if self.result.succeeded:
            return 'green'
        return 'red'

    @property
    def comment_message(self):
        if self.succeeded:
            return "All gooodie good\n\n%s" % self.get_absolute_url()
        else:
            return "Be careful.. the tests failed\n\n%s" % self.get_absolute_url()

    @property
    def queue_object(self):
        return {
            'id': self.pk,
            'branch': self.branch,
            'sha': self.sha,
            'clone_url': self.project.clone_url,
            'owner': self.project.owner,
            'name': self.project.name
        }

    def start(self):
        if hasattr(self, 'result'):
            self.result.delete()

        if not self.approved:
            return BuildResult.create_not_approved(self)

        github.set_commit_status(self, pending=True)

        r = redis.Redis(**settings.REDIS_SETTINGS)
        r.lpush(settings.FRIGG_WORKER_QUEUE, json.dumps(self.queue_object))

        return self

    def handle_worker_report(self, payload):
        BuildResult.create_from_worker_payload(self, payload)

        github.set_commit_status(self)
        if 'comment' in payload and payload['comment']:
            github.comment_on_commit(self, self.comment_message)

        if 'webhooks' in payload:
            for url in payload['webhooks']:
                self.send_webhook(url)

    def send_webhook(self, url):
        return requests.post(url, data=json.dumps({
            'repository': self.project.git_repository,
            'sha': self.sha,
            'build_url': self.get_absolute_url(),
            'pull_request_id': self.pull_request_id,
            'state': self.result.succeeded,
            'return_code': self.result.return_code
        }))

@python_2_unicode_compatible
class BuildResult(models.Model):
    build = models.OneToOneField(Build, related_name='result')
    result_log = models.TextField()
    succeeded = models.BooleanField(default=False)
    return_code = models.CharField(max_length=100)

    def __str__(self):
        return "%s - %s" % (self.build, self.build.build_number)

    @classmethod
    def create_not_approved(cls, build):
        cls.objects.create(build=build, result_log='This project is not approved.', succeeded=False)
        github.set_commit_status(build, error='This project is not approved')

    @classmethod
    def create_from_worker_payload(cls, build, payload):
        result = cls.objects.create(build_id=build.pk, succeeded=True, return_code='',
                                    result_log='')
        return_codes = []
        for r in payload['results']:
            if 'return_code' in r:
                return_codes.append(r['return_code'])
            result.succeeded = 'succeeded' in r and r['succeeded'] and result.succeeded
            result.result_log += cls.create_log_string_for_task(r)
        result.return_code = ",".join([str(code) for code in return_codes])
        result.save()

    @classmethod
    def create_log_string_for_task(cls, result):
        if 'task' in result and 'log' in result and 'return_code' in result:
            return ('Task: %(task)s\n'
                    '\n------------------------------------\n'
                    '%(log)s'
                    '\n------------------------------------\n'
                    'Exited with exit code: %(return_code)s\n\n') % result
        return ''
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import swapper

from accelerator_abstract.models import BaseBusinessProposition

class BusinessProposition(BaseBusinessProposition):
    class Meta(BaseBusinessProposition.Meta):
        pass
<commit_msg>[AC-9381]: Resolve merge conflict<commit_after>from accelerator_abstract.models import BaseBusinessProposition

class BusinessProposition(BaseBusinessProposition):
    class Meta(BaseBusinessProposition.Meta):
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Joel Grand-Guillaume
#    Copyright 2011-2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import sys
import traceback

from openerp.tools.translate import _
import datetime
from openerp.osv.orm import Model
from openerp.osv import fields, osv
from parser import new_bank_statement_parser
from openerp.tools.config import config

class AccountStatementProfil(Model):
    _inherit = "account.statement.profile"

    def get_import_type_selection(self, cr, uid, context=None):
        """This is the method to be inherited for adding the parser"""
        return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')]

    def _get_import_type_selection(self, cr, uid, context=None):
        return self.get_import_type_selection(cr, uid, context=context)

    _columns = {
        'launch_import_completion': fields.boolean(
            "Launch completion after import",
            help="Tic that box to automatically launch the completion "
                 "on each imported file using this profile."),
        'last_import_date': fields.datetime("Last Import Date"),
        #  we remove deprecated as it floods logs in standard/warning level sob...
        'rec_log': fields.text('log', readonly=True),  # Deprecated
        'import_type': fields.selection(
            _get_import_type_selection,
            'Type of import',
            required=True,
            help="Choose here the method by which you want to import bank"
                 "statement for this profile."),
    }

    _defaults = {
                'import_type': 'generic_csvxls_so'
            }

    def _write_extra_statement_lines(
            self, cr, uid, parser, result_row_list, profile, statement_id, context):
        """Insert extra lines after the main statement lines.

        After the main statement lines have been created, you can override this method to create
        extra statement lines.

            :param:    browse_record of the current parser
            :param:    result_row_list: [{'key':value}]
            :param:    profile: browserecord of account.statement.profile
            :param:    statement_id: int/long of the current importing statement ID
            :param:    context: global context
        """
        pass

    def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, context):
        """
        Write the log in the logger

        :param int/long statement_id: ID of the concerned account.bank.statement
        :param int/long num_lines: Number of line that have been parsed
        :return: True
        """
        self.message_post(cr,
                          uid,
                          ids,
                          body=_('Statement ID %s have been imported with %s lines.') %
                                (statement_id, num_lines),
                          context=context)
        return True

    #Deprecated remove on V8
    def prepare_statetement_lines_vals(self, *args, **kwargs):
        return self.prepare_statement_lines_vals(*args, **kwargs)

    def prepare_statement_lines_vals(
            self, cr, uid, parser_vals, account_payable, account_receivable,
            statement_id, context):
        """
        Hook to build the values of a line from the parser returned values. At
        least it fullfill the statement_id. Overide it to add your
        own completion if needed.

        :param dict of vals from parser for account.bank.statement.line (called by
                parser.get_st_line_vals)
        :param int/long account_payable: ID of the receivable account to use
        :param int/long account_receivable: ID of the payable account to use
        :param int/long statement_id: ID of the concerned account.bank.statement
        :return: dict of vals that will be passed to create method of statement line.
        """
        statement_obj = self.pool.get('account.bank.statement')
        statement_line_obj = self.pool['account.bank.statement.line']
        values = parser_vals
        values['statement_id'] = statement_id
        date = values.get('date')
        period_memoizer = context.get('period_memoizer')
        if not period_memoizer:
            period_memoizer = {}
            context['period_memoizer'] = period_memoizer
        if period_memoizer.get(date):
            values['period_id'] = period_memoizer[date]
        else:
            # This is awfully slow...
            periods = self.pool.get('account.period').find(cr, uid,
                                                           dt=values.get('date'),
                                                           context=context)
            values['period_id'] = periods[0]
            period_memoizer[date] = periods[0]
        values = statement_line_obj._add_missing_default_values(cr, uid, values, context)
        return values

    def prepare_statement_vals(self, cr, uid, profile_id, result_row_list, parser, context):
        """
        Hook to build the values of the statement from the parser and
        the profile.
        """
        vals = {'profile_id': profile_id}
        vals.update(parser.get_st_vals())
        return vals

    def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None):
        """
        Create a bank statement with the given profile and parser. It will fullfill the bank statement
        with the values of the file providen, but will not complete data (like finding the partner, or
        the right account). This will be done in a second step with the completion rules.

        :param int/long profile_id: ID of the profile used to import the file
        :param filebuffer file_stream: binary of the providen file
        :param char: ftype represent the file exstension (csv by default)
        :return: ID of the created account.bank.statemênt
        """
        statement_obj = self.pool.get('account.bank.statement')
        statement_line_obj = self.pool.get('account.bank.statement.line')
        attachment_obj = self.pool.get('ir.attachment')
        prof_obj = self.pool.get("account.statement.profile")
        if not profile_id:
            raise osv.except_osv(_("No Profile!"),
                                 _("You must provide a valid profile to import a bank statement!"))
        prof = prof_obj.browse(cr, uid, profile_id, context=context)

        parser = new_bank_statement_parser(prof.import_type, ftype=ftype)
        result_row_list = parser.parse(file_stream)
        # Check all key are present in account.bank.statement.line!!
        if not result_row_list:
            raise osv.except_osv(_("Nothing to import"),
                                 _("The file is empty"))
        parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys()
        for col in parsed_cols:
            if col not in statement_line_obj._columns:
                raise osv.except_osv(_("Missing column!"),
                                     _("Column %s you try to import is not "
                                       "present in the bank statement line!") % col)

        statement_vals = self.prepare_statement_vals(cr, uid, prof.id, result_row_list, parser, context)
        statement_id = statement_obj.create(cr, uid,
                                            statement_vals,
                                            context=context)

        if prof.receivable_account_id:
            account_receivable = account_payable = prof.receivable_account_id.id
        else:
            account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(
                                                       cr, uid, context)
        try:
            # Record every line in the bank statement
            statement_store = []
            for line in result_row_list:
                parser_vals = parser.get_st_line_vals(line)
                values = self.prepare_statement_lines_vals(
                    cr, uid, parser_vals, account_payable, account_receivable, statement_id,
                    context)
                statement_store.append(values)
            # Hack to bypass ORM poor perfomance. Sob...
            statement_line_obj._insert_lines(cr, uid, statement_store, context=context)

            self._write_extra_statement_lines(
                cr, uid, parser, result_row_list, prof, statement_id, context)
            # Trigger store field computation if someone has better idea
            start_bal = statement_obj.read(
                cr, uid, statement_id, ['balance_start'], context=context)
            start_bal = start_bal['balance_start']
            statement_obj.write(cr, uid, [statement_id], {'balance_start': start_bal})

            attachment_data = {
                'name': 'statement file',
                'datas': file_stream,
                'datas_fname': "%s.%s" % (datetime.datetime.now().date(), ftype),
                'res_model': 'account.bank.statement',
                'res_id': statement_id,
            }
            attachment_obj.create(cr, uid, attachment_data, context=context)

            # If user ask to launch completion at end of import, do it!
            if prof.launch_import_completion:
                statement_obj.button_auto_completion(cr, uid, [statement_id], context)

            # Write the needed log infos on profile
            self.write_logs_after_import(cr, uid, prof.id,
                                         statement_id,
                                         len(result_row_list),
                                         context)

        except Exception:
            error_type, error_value, trbk = sys.exc_info()
            st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value)
            st += ''.join(traceback.format_tb(trbk, 30))
            #TODO we should catch correctly the exception with a python
            #Exception and only re-catch some special exception.
            #For now we avoid re-catching error in debug mode
            if config['debug_mode']:
                raise
            raise osv.except_osv(_("Statement import error"),
                                 _("The statement cannot be created: %s") % st)
        return statement_id

class AccountBankStatementLine(Model):
    _inherit = "account.bank.statement.line"

    _columns = {
        'account_id': fields.many2one('account.account','Account'),
    }

    _defaults = {
        'account_id': False,
    }

<commit_msg>PEP-8<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Joel Grand-Guillaume
#    Copyright 2011-2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import sys
import traceback

from openerp.tools.translate import _
import datetime
from openerp.osv.orm import Model
from openerp.osv import fields, osv
from parser import new_bank_statement_parser
from openerp.tools.config import config

class AccountStatementProfil(Model):
    _inherit = "account.statement.profile"

    def get_import_type_selection(self, cr, uid, context=None):
        """This is the method to be inherited for adding the parser"""
        return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')]

    def _get_import_type_selection(self, cr, uid, context=None):
        return self.get_import_type_selection(cr, uid, context=context)

    _columns = {
        'launch_import_completion': fields.boolean(
            "Launch completion after import",
            help="Tic that box to automatically launch the completion "
                 "on each imported file using this profile."),
        'last_import_date': fields.datetime("Last Import Date"),
        #  we remove deprecated as it floods logs in standard/warning level sob...
        'rec_log': fields.text('log', readonly=True),  # Deprecated
        'import_type': fields.selection(
            _get_import_type_selection,
            'Type of import',
            required=True,
            help="Choose here the method by which you want to import bank"
                 "statement for this profile."),
    }

    _defaults = {
                'import_type': 'generic_csvxls_so'
            }

    def _write_extra_statement_lines(
            self, cr, uid, parser, result_row_list, profile, statement_id, context):
        """Insert extra lines after the main statement lines.

        After the main statement lines have been created, you can override this method to create
        extra statement lines.

            :param:    browse_record of the current parser
            :param:    result_row_list: [{'key':value}]
            :param:    profile: browserecord of account.statement.profile
            :param:    statement_id: int/long of the current importing statement ID
            :param:    context: global context
        """
        pass

    def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, context):
        """
        Write the log in the logger

        :param int/long statement_id: ID of the concerned account.bank.statement
        :param int/long num_lines: Number of line that have been parsed
        :return: True
        """
        self.message_post(cr,
                          uid,
                          ids,
                          body=_('Statement ID %s have been imported with %s lines.') %
                                (statement_id, num_lines),
                          context=context)
        return True

    #Deprecated remove on V8
    def prepare_statetement_lines_vals(self, *args, **kwargs):
        return self.prepare_statement_lines_vals(*args, **kwargs)

    def prepare_statement_lines_vals(
            self, cr, uid, parser_vals, account_payable, account_receivable,
            statement_id, context):
        """
        Hook to build the values of a line from the parser returned values. At
        least it fullfill the statement_id. Overide it to add your
        own completion if needed.

        :param dict of vals from parser for account.bank.statement.line (called by
                parser.get_st_line_vals)
        :param int/long account_payable: ID of the receivable account to use
        :param int/long account_receivable: ID of the payable account to use
        :param int/long statement_id: ID of the concerned account.bank.statement
        :return: dict of vals that will be passed to create method of statement line.
        """
        statement_line_obj = self.pool['account.bank.statement.line']
        values = parser_vals
        values['statement_id'] = statement_id
        date = values.get('date')
        period_memoizer = context.get('period_memoizer')
        if not period_memoizer:
            period_memoizer = {}
            context['period_memoizer'] = period_memoizer
        if period_memoizer.get(date):
            values['period_id'] = period_memoizer[date]
        else:
            # This is awfully slow...
            periods = self.pool.get('account.period').find(cr, uid,
                                                           dt=values.get('date'),
                                                           context=context)
            values['period_id'] = periods[0]
            period_memoizer[date] = periods[0]
        values = statement_line_obj._add_missing_default_values(cr, uid, values, context)
        return values

    def prepare_statement_vals(self, cr, uid, profile_id, result_row_list, parser, context):
        """
        Hook to build the values of the statement from the parser and
        the profile.
        """
        vals = {'profile_id': profile_id}
        vals.update(parser.get_st_vals())
        return vals

    def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None):
        """
        Create a bank statement with the given profile and parser. It will fullfill the bank statement
        with the values of the file providen, but will not complete data (like finding the partner, or
        the right account). This will be done in a second step with the completion rules.

        :param int/long profile_id: ID of the profile used to import the file
        :param filebuffer file_stream: binary of the providen file
        :param char: ftype represent the file exstension (csv by default)
        :return: ID of the created account.bank.statemênt
        """
        statement_obj = self.pool.get('account.bank.statement')
        statement_line_obj = self.pool.get('account.bank.statement.line')
        attachment_obj = self.pool.get('ir.attachment')
        prof_obj = self.pool.get("account.statement.profile")
        if not profile_id:
            raise osv.except_osv(_("No Profile!"),
                                 _("You must provide a valid profile to import a bank statement!"))
        prof = prof_obj.browse(cr, uid, profile_id, context=context)

        parser = new_bank_statement_parser(prof.import_type, ftype=ftype)
        result_row_list = parser.parse(file_stream)
        # Check all key are present in account.bank.statement.line!!
        if not result_row_list:
            raise osv.except_osv(_("Nothing to import"),
                                 _("The file is empty"))
        parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys()
        for col in parsed_cols:
            if col not in statement_line_obj._columns:
                raise osv.except_osv(_("Missing column!"),
                                     _("Column %s you try to import is not "
                                       "present in the bank statement line!") % col)

        statement_vals = self.prepare_statement_vals(cr, uid, prof.id, result_row_list, parser, context)
        statement_id = statement_obj.create(cr, uid,
                                            statement_vals,
                                            context=context)

        if prof.receivable_account_id:
            account_receivable = account_payable = prof.receivable_account_id.id
        else:
            account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(
                                                       cr, uid, context)
        try:
            # Record every line in the bank statement
            statement_store = []
            for line in result_row_list:
                parser_vals = parser.get_st_line_vals(line)
                values = self.prepare_statement_lines_vals(
                    cr, uid, parser_vals, account_payable, account_receivable, statement_id,
                    context)
                statement_store.append(values)
            # Hack to bypass ORM poor perfomance. Sob...
            statement_line_obj._insert_lines(cr, uid, statement_store, context=context)

            self._write_extra_statement_lines(
                cr, uid, parser, result_row_list, prof, statement_id, context)
            # Trigger store field computation if someone has better idea
            start_bal = statement_obj.read(
                cr, uid, statement_id, ['balance_start'], context=context)
            start_bal = start_bal['balance_start']
            statement_obj.write(cr, uid, [statement_id], {'balance_start': start_bal})

            attachment_data = {
                'name': 'statement file',
                'datas': file_stream,
                'datas_fname': "%s.%s" % (datetime.datetime.now().date(), ftype),
                'res_model': 'account.bank.statement',
                'res_id': statement_id,
            }
            attachment_obj.create(cr, uid, attachment_data, context=context)

            # If user ask to launch completion at end of import, do it!
            if prof.launch_import_completion:
                statement_obj.button_auto_completion(cr, uid, [statement_id], context)

            # Write the needed log infos on profile
            self.write_logs_after_import(cr, uid, prof.id,
                                         statement_id,
                                         len(result_row_list),
                                         context)

        except Exception:
            error_type, error_value, trbk = sys.exc_info()
            st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value)
            st += ''.join(traceback.format_tb(trbk, 30))
            #TODO we should catch correctly the exception with a python
            #Exception and only re-catch some special exception.
            #For now we avoid re-catching error in debug mode
            if config['debug_mode']:
                raise
            raise osv.except_osv(_("Statement import error"),
                                 _("The statement cannot be created: %s") % st)
        return statement_id

class AccountBankStatementLine(Model):
    _inherit = "account.bank.statement.line"

    _columns = {
        'account_id': fields.many2one('account.account', 'Account'),
    }

    _defaults = {
        'account_id': False,
    }
<|endoftext|>


<commit_before>"""Example of a master script with robot interface.

"""

import time
import cv2
import ev3control
from ev3control import Robot
from brain.controllers import move_to_brick_simple, \
    move_to_brick_blind_and_grip, rotation_search_brick, rotation_search_box, move_to_box_simple
from brain.core import State
from brain.core import main_loop

print("Creating robot...")
robot = Robot({

    "leftMotor": "LargeMotor('outA')",
    "rightMotor": "LargeMotor('outB')",
    "gripper": "MediumMotor('outC')",
    "elevator": "LargeMotor('outD')"

}, cap=cv2.VideoCapture(0))

def stop_after_dec(controller):

    def func(robot,frame, **kwargs):
        controller(robot,frame)
        robot.stop_all_motors()
        return "FINAL_STATE",frame, {}
    return func

# Define the state graph, we can do this better, currently each method returns the next state name
states = [
    State(name="MOVE_TO_BRICK_BLIND_AND_GRIP", act=stop_after_dec(move_to_brick_blind_and_grip)),
    State(name="FINAL_STATE", act=lambda x,y:None)
]
state_dict = {}
for state in states:
    state_dict[state.name] = state

start_state = states[0]
main_loop(robot, start_state, state_dict)

<commit_msg>Fixed gripping benchmark script<commit_after>"""Example of a master script with robot interface.

"""

import time
import cv2
import ev3control
from ev3control import Robot
from brain.controllers import move_to_brick_simple, \
    move_to_brick_blind_and_grip, rotation_search_brick, rotation_search_box, move_to_box_simple
from brain.core import State
from brain.core import main_loop

print("Creating robot...")
robot = Robot({

    "leftMotor": "LargeMotor('outA')",
    "rightMotor": "LargeMotor('outB')",
    "gripper": "MediumMotor('outC')",
    "elevator": "LargeMotor('outD')"

}, cap=cv2.VideoCapture(1))

def stop_after_dec(controller):

    def func(robot,frame, **kwargs):
        controller(robot,frame)
        robot.stop_all_motors()
        return "FINAL_STATE",frame, {}
    return func

# Define the state graph, we can do this better, currently each method returns the next state name
states = [
    State(name="MOVE_TO_BRICK_BLIND_AND_GRIP", act=move_to_brick_blind_and_grip),
    State(name="FINAL_STATE", act=lambda x,y:None)
]
state_dict = {}
for state in states:
    state_dict[state.name] = state

start_state = states[0]
main_loop(robot, start_state, state_dict)

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Generates bash completion file for MRtrix commands
# MRtrix needs to be built prior to running this
#
# Author: Rami Tabbara
#

from __future__ import print_function
import getopt, sys, os, re, subprocess

def usage():
  print ('''
Usage:	%s -m [dir_path] -c [file_path]\n
	-m, --mrtrix_commands_dir [dir path]: The directory where the mrtrix executable commands reside.
	-c, --completion_path [file path]: The generated bash completion file path.
''' % (sys.argv[0]))

def main(argv):
	try:
		opts, args = getopt.getopt(argv, "m:c:", ["mrtrix_commands_dir=", "completion_path="])
		if not opts or len(opts) != 2:
			raise getopt.GetoptError('Incorrect number of arguments')
	except getopt.GetoptError:
		usage()
		sys.exit(2)

	commands_dir = ""
	completion_path = ""
	commands = []

	for opt, arg in opts:
		if opt in ("-m", "--mrtrix_commands_dir"):
			commands_dir = arg
		elif opt in ("-c", "--completion_path"):
			completion_path = arg

	if not commands_dir or not os.path.isdir (commands_dir) or not completion_path:
		usage()
		sys.exit (2)

	# Filter out any non executable files or debug commands
	commands = [comm for comm in os.listdir (commands_dir) if os.access ( os.path.join( commands_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]

	if not commands:
		sys.exit (0)

	parse_commands (commands_dir, completion_path, commands)

###########################################################################
#                         PARSE FUNCTION
###########################################################################

def parse_commands (commands_dir, completion_path, commands):

	completion_file = open (completion_path, 'w+')

###########################################################################
#                         INNER PARSE FUNCTIONS
###########################################################################

	def flush_option_num_args ():
		if not current_option:
			print ('\t_%s_max_num_args() { echo %s; }\n' % (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_max_num_args() { echo %s; }\n' % (command, current_option, current_num_of_option_args), file = completion_file)

	def flush_option_arg ():
		if not current_option:
			print ('\t_%s_arg_%s()' % (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_arg_%s()' % (command, current_option, current_num_of_option_args), file = completion_file)

		print ('\t{ echo "%s"; }\n' % (current_arg_choices) , file = completion_file)

	def	flush_option_arg_allows_multiple ():
		if not current_option:
			print ('\t_%s_arg_%s_allows_multiple()' %  (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_arg_%s_allows_multiple()' % (command, current_option, current_num_of_option_args), file = completion_file)

		print ('\t{ echo %s; }\n' % (current_arg_allows_multiple), file = completion_file)

	def parse_option_arg_choices (arg_type, choices = []):
		arg_choices = ""

		if arg_type == 'CHOICE':
			for choice in choices:
				choice = choice.rstrip ()
				# End of choice lists are generally delimited by -1
				# however tckgen choices are delimited by 2 (?)
				if choice in ['-1', '2']:
					break;
				else:
					arg_choices +=  " " + choice
		elif arg_type == 'IMAGEIN':
			arg_choices ='`ls *.{mih,mif,nii,dcm,msf,hdr,mgh} 2> /dev/null | tr "\n" " "`'
		elif arg_type == 'FILEIN':
			arg_choices ='`ls 2> /dev/null | grep -E "*\.[^[:space:]]+"`'
		elif arg_type in ['FLOAT', 'INT']:
			# Use default value
			arg_choices = choices[2].rstrip ()
		return arg_choices

###########################################################################
#                         END INNER PARSE FUNCTIONS
###########################################################################

	for command in commands:
		single_dash_options = ""
		double_dash_options = ""
		current_option = ""
		current_num_of_option_args = 0
		current_arg = ""
		current_arg_optional = -1
		current_arg_allows_multiple = -1
		current_arg_type = ""
		current_arg_choices = ""

		print ('''
_%s()
{
''' % (command), file = completion_file)

		try:
			process = subprocess.Popen ([ os.path.join( commands_dir, command ), '__print_full_usage__'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
			for line in process.stdout:
				words = line.split (' ')

				if not words:
					continue

				if words[0] == 'OPTION':
					flush_option_num_args ()
					current_arg = ""
					current_arg_type = ""
					current_option = words[1]
					single_dash_options += ' -' + current_option
					double_dash_options += ' --' + current_option
					current_num_of_option_args = 0
				elif words[0] == 'ARGUMENT':
					current_arg = words[1]
					current_arg_optional = words[2]
					current_arg_allows_multiple = words[3]
					current_arg_type = words[4].rstrip ()
					current_arg_choices = parse_option_arg_choices (current_arg_type, words[5:])
					flush_option_arg_allows_multiple ()
					flush_option_arg ()
					current_num_of_option_args += 1
		except:
			pass

		flush_option_num_args ()

		print ('''
	current_option=""
	current_max_args=0
	current_arg_index=0;
	consumed_option_tokens=0;
	current_num_consumed_option_args=0;
	first_option_consumed=0;

	COMPREPLY=();
	cur="${COMP_WORDS[COMP_CWORD]}";

	if [[ "$cur" == --* ]]; then
		COMPREPLY=( $( compgen -W "%s" -- "$cur" ) );
	elif [[ "$cur" == -* ]]; then
		COMPREPLY=( $( compgen -W "%s" -- "$cur" ) );
	else
		current_option=""
		word=""

		for ((i=${#COMP_WORDS[@]}-2; i>=0; i--)); do
  			word="${COMP_WORDS[$i]}"
			if [[ "$word" == -* ]]; then
				option=$(echo $word | sed -E "s/-//g");
				arg_index=$(( $COMP_CWORD - $i - 1));
				max_args=$(_%s_${option}_max_num_args);

				current_num_consumed_option_args=$((current_num_consumed_option_args > max_args ? max_args : current_num_consumed_option_args))
				consumed_option_tokens=$(($current_consumed_option_tokens + $current_num_consumed_option_args + 1))
				current_num_consumed_option_args=0;

				if (( first_option_consumed == 0 )); then
					current_option="$option";
					current_arg_index=$arg_index;
					current_max_args=$max_args;
					first_option_consumed=1;
				fi

			else
				current_num_consumed_option_args=$(($current_num_option_args + 1));
			fi
		done

		if [[ "$current_option" != "" ]] && (( $current_arg_index >= 0 && $current_arg_index < $current_max_args )); then
			COMPREPLY=( $(_%s_${current_option}_arg_${current_arg_index}));
		else
			current_arg_index=$(($COMP_CWORD - $consumed_option_tokens - 1));
			current_max_args=$(_%s_max_num_args);

			# Check whether any of our arguments accepts multiple inputs
			# If so, adjust the argument index to be the min. of these
			for ((i=${current_max_args} - 1; i>=0; i--)); do
				if (( $(_%s_arg_${i}_allows_multiple) == 1 )); then
					current_arg_index=$((current_arg_index > i ? i : current_arg_index))
				fi
			done

			if (( $current_arg_index >= 0 && $current_arg_index < $current_max_args )); then
				COMPREPLY=( $(_%s_arg_${current_arg_index}));
			fi
		fi

		COMPREPLY=( $( for i in ${COMPREPLY[@]} ; do echo $i ; done | grep "^${cur}" ) )
	fi
}
complete -F _%s %s \n''' % (double_dash_options, single_dash_options, command, command, command, command, command, command, command), file = completion_file)

	completion_file.close ()

if __name__ == "__main__":
	main(sys.argv[1:])
<commit_msg>Proposed fix for #370 Bash Auto Complete folder names, nii.gz and mif.gz<commit_after>#!/usr/bin/env python
#
# Generates bash completion file for MRtrix commands
# MRtrix needs to be built prior to running this
#
# Author: Rami Tabbara
#

from __future__ import print_function
import getopt, sys, os, re, subprocess

def usage():
  print ('''
Usage:	%s -m [dir_path] -c [file_path]\n
	-m, --mrtrix_commands_dir [dir path]: The directory where the mrtrix executable commands reside.
	-c, --completion_path [file path]: The generated bash completion file path.
''' % (sys.argv[0]))

def main(argv):
	try:
		opts, args = getopt.getopt(argv, "m:c:", ["mrtrix_commands_dir=", "completion_path="])
		if not opts or len(opts) != 2:
			raise getopt.GetoptError('Incorrect number of arguments')
	except getopt.GetoptError:
		usage()
		sys.exit(2)

	commands_dir = ""
	completion_path = ""
	commands = []

	for opt, arg in opts:
		if opt in ("-m", "--mrtrix_commands_dir"):
			commands_dir = arg
		elif opt in ("-c", "--completion_path"):
			completion_path = arg

	if not commands_dir or not os.path.isdir (commands_dir) or not completion_path:
		usage()
		sys.exit (2)

	# Filter out any non executable files or debug commands
	commands = [comm for comm in os.listdir (commands_dir) if os.access ( os.path.join( commands_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]

	if not commands:
		sys.exit (0)

	parse_commands (commands_dir, completion_path, commands)

###########################################################################
#                         PARSE FUNCTION
###########################################################################

def parse_commands (commands_dir, completion_path, commands):

	completion_file = open (completion_path, 'w+')

###########################################################################
#                         INNER PARSE FUNCTIONS
###########################################################################

	def flush_option_num_args ():
		if not current_option:
			print ('\t_%s_max_num_args() { echo %s; }\n' % (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_max_num_args() { echo %s; }\n' % (command, current_option, current_num_of_option_args), file = completion_file)

	def flush_option_arg ():
		if not current_option:
			print ('\t_%s_arg_%s()' % (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_arg_%s()' % (command, current_option, current_num_of_option_args), file = completion_file)

		print ('\t{ echo "%s"; }\n' % (current_arg_choices) , file = completion_file)

	def	flush_option_arg_allows_multiple ():
		if not current_option:
			print ('\t_%s_arg_%s_allows_multiple()' %  (command, current_num_of_option_args), file = completion_file)
		else:
			print ('\t_%s_%s_arg_%s_allows_multiple()' % (command, current_option, current_num_of_option_args), file = completion_file)

		print ('\t{ echo %s; }\n' % (current_arg_allows_multiple), file = completion_file)

	def parse_option_arg_choices (arg_type, choices = []):
		arg_choices = ""

		if arg_type == 'CHOICE':
			for choice in choices:
				choice = choice.rstrip ()
				# End of choice lists are generally delimited by -1
				# however tckgen choices are delimited by 2 (?)
				if choice in ['-1', '2']:
					break;
				else:
					arg_choices +=  " " + choice
		elif arg_type == 'IMAGEIN':
			arg_choices ='`eval ls $1*.{mih,mif,nii,dcm,msf,hdr,mgh,nii.gz,mif.gz} 2> /dev/null | tr "\n" " "`'
		elif arg_type == 'FILEIN':
			arg_choices ='`eval ls "$1*" 2> /dev/null | grep -E "$1*\.[^[:space:]]+"`'
		#elif arg_type in ['FLOAT', 'INT']:
		else:
			arg_choices = "__empty__"

		return arg_choices

###########################################################################
#                         END INNER PARSE FUNCTIONS
###########################################################################

	for command in commands:
		single_dash_options = ""
		double_dash_options = ""
		current_option = ""
		current_num_of_option_args = 0
		current_arg = ""
		current_arg_optional = -1
		current_arg_allows_multiple = -1
		current_arg_type = ""
		current_arg_choices = ""

		print ('''
_%s()
{
''' % (command), file = completion_file)

		try:
			process = subprocess.Popen ([ os.path.join( commands_dir, command ), '__print_full_usage__'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
			for line in process.stdout:
				words = line.split (' ')

				if not words:
					continue

				if words[0] == 'OPTION':
					flush_option_num_args ()
					current_arg = ""
					current_arg_type = ""
					current_option = words[1]
					single_dash_options += ' -' + current_option
					double_dash_options += ' --' + current_option
					current_num_of_option_args = 0
				elif words[0] == 'ARGUMENT':
					current_arg = words[1]
					current_arg_optional = words[2]
					current_arg_allows_multiple = words[3]
					current_arg_type = words[4].rstrip ()
					current_arg_choices = parse_option_arg_choices (current_arg_type, words[5:])
					flush_option_arg_allows_multiple ()
					flush_option_arg ()
					current_num_of_option_args += 1
		except:
			pass

		flush_option_num_args ()

		print ('''
	current_option=""
	current_max_args=0
	current_arg_index=0;
	consumed_option_tokens=0;
	current_num_consumed_option_args=0;
	first_option_consumed=0;

	COMPREPLY=();
	cur="${COMP_WORDS[COMP_CWORD]}";
	cur_filtered=`echo "${COMP_WORDS[COMP_CWORD]}" | sed 's/\..*$//g'`;

	if [[ "$cur" == --* ]]; then
		COMPREPLY=( $( compgen -W "%s" -- "$cur" ) );
	elif [[ "$cur" == -* ]]; then
		COMPREPLY=( $( compgen -W "%s" -- "$cur" ) );
	else
		current_option=""
		word=""
		array=""

		for ((i=${#COMP_WORDS[@]}-2; i>=0; i--)); do
  			word="${COMP_WORDS[$i]}"
			if [[ "$word" == -* ]]; then
				option=$(echo $word | sed -E "s/-//g");
				arg_index=$(( $COMP_CWORD - $i - 1));
				max_args=$(_%s_${option}_max_num_args);

				current_num_consumed_option_args=$((current_num_consumed_option_args > max_args ? max_args : current_num_consumed_option_args))
				consumed_option_tokens=$(($current_consumed_option_tokens + $current_num_consumed_option_args + 1))
				current_num_consumed_option_args=0;

				if (( first_option_consumed == 0 )); then
					current_option="$option";
					current_arg_index=$arg_index;
					current_max_args=$max_args;
					first_option_consumed=1;
				fi

			else
				current_num_consumed_option_args=$(($current_num_option_args + 1));
			fi
		done

		if [[ "$current_option" != "" ]] && (( $current_arg_index >= 0 && $current_arg_index < $current_max_args )); then
			array=`_%s_${current_option}_arg_${current_arg_index} $cur_filtered`
		else
			current_arg_index=$(($COMP_CWORD - $consumed_option_tokens - 1));
			current_max_args=$(_%s_max_num_args);

			# Check whether any of our arguments accepts multiple inputs
			# If so, adjust the argument index to be the min. of these
			for ((i=${current_max_args} - 1; i>=0; i--)); do
				if (( $(_%s_arg_${i}_allows_multiple) == 1 )); then
					current_arg_index=$((current_arg_index > i ? i : current_arg_index))
				fi
			done

			if (( $current_arg_index >= 0 && $current_arg_index < $current_max_args )); then
				array=`_%s_arg_${current_arg_index} $cur_filtered`;
			fi
		fi

		if [[ "$array" == "__empty__" ]]; then
			COMPREPLY=()
			compopt +o dirnames %s;
		else
			mapfile -t COMPREPLY < <( compgen -W "$array" -- "$cur" )
			compopt -o dirnames %s;
		fi
	fi
}
complete -F _%s %s \n''' % (double_dash_options, single_dash_options, command, command, command, command, command, command, command, command, command), file = completion_file)

	completion_file.close ()

if __name__ == "__main__":
	main(sys.argv[1:])
<|endoftext|>


<commit_before>'''
Created on Jan 11, 2016

@author: Martin Koerner, Tatiana Sennikova
'''

from SPARQLWrapper import SPARQLWrapper, JSON
from SPARQLWrapper.SPARQLExceptions import QueryBadFormed
import json
import argparse
from utils import dbpedia_mapping, country_lookup, majority_voting
import urllib2
import socket

# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts geo locations from a list of wikipedia articles given in JSON.')
parser.add_argument('input',
                           help='a file path to a JSON file containing wikipedia article names')
parser.add_argument("--output", dest="output", metavar='output path', type=str)
parser.add_argument("--language", dest="language", metavar='two-letter country code', type=str, help="on of the language editions of dbpedia (default: en):", required=True)
parser.add_argument("--threshold", dest="threshold", metavar='threshold for majority voting', type=float, help="absolute threshold for majority voting on coordinates (default: 0.1)", required=True)

args = parser.parse_args()

inputfile_path=args.input
outputfile_path=args.output
language = args.language
threshold = args.threshold

print "running wikipedia_location_extraction"

# Convert coordinates to decimals
def dms2dd(degrees, minutes, seconds, direction):
    dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60);
    if direction == 'S' or direction == 'W':
        dd *= -1
    return dd

def load_simple_json(filename):
    with open(filename, 'r') as f:
#        return json.load(f)
        return json.load(f)

# initialize sparql querier
# get mapping from two-letter country code to dbpedia endpoint URL
dbpedia_url = dbpedia_mapping.language_to_dbpedia_url(language)
sparql = SPARQLWrapper(dbpedia_url)
sparql.setTimeout(1000)
sparql.setReturnFormat(JSON)

article_language_dictionary={}

def query_location(article):

    article_url = '<http://'+ language + '.wikipedia.org/wiki/'+article + '>'
    location = None

    # SPARQL query that
    namespaces = """
    PREFIX dbo: <http://dbpedia.org/resource/classes#>
    PREFIX dbp: <http://dbpedia.org/property/>
    """

    location_query = """
    {
        SELECT * WHERE {
            ?match dbp:latitude ?lat .
            ?match dbp:longitude ?long
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match geo:lat ?lat .
            ?match geo:long ?long
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDeg ?latDeg .
            ?match dbp:latMin ?latMin .
            ?match dbp:latSec ?latSec .
            ?match dbp:lonDeg ?lonDeg .
            ?match dbp:lonMin ?lonMin .
            ?match dbp:lonSec ?lonSec
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDeg ?latDeg .
            ?match dbp:latMin ?latMin .
            ?match dbp:latSec ?latSec .
            ?match dbp:lonDeg ?lonDeg .
            ?match dbp:lonMin ?lonMin .
            ?match dbp:lonSec ?lonSec .
            ?match dbp:latDir ?latDir .
            ?match dbp:lonDir ?lonDir
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDegrees ?latDeg .
            ?match dbp:latMinutes ?latMin .
            ?match dbp:latSeconds ?latSec .
            ?match dbp:longDegrees ?lonDeg .
            ?match dbp:longMinutes ?lonMin .
            ?match dbp:longSeconds ?lonSec .
            ?match dbp:latDirection ?latDir .
            ?match dbp:longDirection ?lonDir

        }
    }

    UNION
    {
        SELECT * WHERE {
            ?match dbp:latd ?lat .
            ?match dbp:longd ?long
        }
    }

    """
    query_string = namespaces + """
    SELECT * WHERE {
        ?match foaf:isPrimaryTopicOf """+article_url+""" .
        """+location_query+"""
    }
    """

    query_string_with_offset = query_string

    sparql.setQuery(query_string_with_offset)

    try:
        results = sparql.query().convert()
    except QueryBadFormed:
        print "SPARQL query bad formed: " + query_string_with_offset
        return None
    except urllib2.HTTPError:
        print "HTTP Error 502: " + article
        return None
    except urllib2.URLError:
        print "Network is unreachable while working on: " + article
        return None
    except socket.timeout:
        print "Query timed out for: " + article
        return None

    if len(results["results"]["bindings"]) > 0:
        coordinates_array = []
        for result in results["results"]["bindings"]:
            if  "lat" in result:
                latitude = result["lat"]["value"]
                longitude = result["long"]["value"]
            else:
                lat_dir=0
                if "latDir" in result:
                    lat_dir = result["latDir"]["value"]

                latitude = dms2dd(result["latDeg"]["value"], result["latMin"]["value"], result["latSec"]["value"], lat_dir)
                longitude = dms2dd(result["lonDeg"]["value"], result["lonMin"]["value"], result["lonSec"]["value"], lat_dir)

            latitude = float(latitude)
            longitude = float(longitude)
            coordinates_array.append((latitude,longitude))

        majority_vote =  majority_voting.vote(coordinates_array,threshold)
        location = country_lookup.get_country(majority_vote[0],majority_vote[1])

    return location

lang_dump = load_simple_json(inputfile_path)

article_count = 0
# Main loop
for article in lang_dump:
    article_with_underscores = article.replace(' ', '_')
    location=query_location(article_with_underscores)
    if location is not None:
        print article + " --> " +location
        article_language_dictionary[article]=location

    article_count += 1

    print article_count

with open(outputfile_path, 'w') as f:
    json.dump(article_language_dictionary, f, indent=4, sort_keys=True)
    print "File was stored successfully"<commit_msg>Remove print outs in wikipedia_location_extraction<commit_after>'''
Created on Jan 11, 2016

@author: Martin Koerner, Tatiana Sennikova
'''

from SPARQLWrapper import SPARQLWrapper, JSON
from SPARQLWrapper.SPARQLExceptions import QueryBadFormed
import json
import argparse
from utils import dbpedia_mapping, country_lookup, majority_voting
import urllib2
import socket

# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts geo locations from a list of wikipedia articles given in JSON.')
parser.add_argument('input',
                           help='a file path to a JSON file containing wikipedia article names')
parser.add_argument("--output", dest="output", metavar='output path', type=str)
parser.add_argument("--language", dest="language", metavar='two-letter country code', type=str, help="on of the language editions of dbpedia (default: en):", required=True)
parser.add_argument("--threshold", dest="threshold", metavar='threshold for majority voting', type=float, help="absolute threshold for majority voting on coordinates (default: 0.1)", required=True)

args = parser.parse_args()

inputfile_path=args.input
outputfile_path=args.output
language = args.language
threshold = args.threshold

print "running wikipedia_location_extraction"

# Convert coordinates to decimals
def dms2dd(degrees, minutes, seconds, direction):
    dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60);
    if direction == 'S' or direction == 'W':
        dd *= -1
    return dd

def load_simple_json(filename):
    with open(filename, 'r') as f:
#        return json.load(f)
        return json.load(f)

# initialize sparql querier
# get mapping from two-letter country code to dbpedia endpoint URL
dbpedia_url = dbpedia_mapping.language_to_dbpedia_url(language)
sparql = SPARQLWrapper(dbpedia_url)
sparql.setTimeout(1000)
sparql.setReturnFormat(JSON)

article_language_dictionary={}

def query_location(article):

    article_url = '<http://'+ language + '.wikipedia.org/wiki/'+article + '>'
    location = None

    # SPARQL query that
    namespaces = """
    PREFIX dbo: <http://dbpedia.org/resource/classes#>
    PREFIX dbp: <http://dbpedia.org/property/>
    """

    location_query = """
    {
        SELECT * WHERE {
            ?match dbp:latitude ?lat .
            ?match dbp:longitude ?long
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match geo:lat ?lat .
            ?match geo:long ?long
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDeg ?latDeg .
            ?match dbp:latMin ?latMin .
            ?match dbp:latSec ?latSec .
            ?match dbp:lonDeg ?lonDeg .
            ?match dbp:lonMin ?lonMin .
            ?match dbp:lonSec ?lonSec
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDeg ?latDeg .
            ?match dbp:latMin ?latMin .
            ?match dbp:latSec ?latSec .
            ?match dbp:lonDeg ?lonDeg .
            ?match dbp:lonMin ?lonMin .
            ?match dbp:lonSec ?lonSec .
            ?match dbp:latDir ?latDir .
            ?match dbp:lonDir ?lonDir
        }
    }
    UNION
    {
        SELECT * WHERE {
            ?match dbp:latDegrees ?latDeg .
            ?match dbp:latMinutes ?latMin .
            ?match dbp:latSeconds ?latSec .
            ?match dbp:longDegrees ?lonDeg .
            ?match dbp:longMinutes ?lonMin .
            ?match dbp:longSeconds ?lonSec .
            ?match dbp:latDirection ?latDir .
            ?match dbp:longDirection ?lonDir

        }
    }

    UNION
    {
        SELECT * WHERE {
            ?match dbp:latd ?lat .
            ?match dbp:longd ?long
        }
    }

    """
    query_string = namespaces + """
    SELECT * WHERE {
        ?match foaf:isPrimaryTopicOf """+article_url+""" .
        """+location_query+"""
    }
    """

    query_string_with_offset = query_string

    sparql.setQuery(query_string_with_offset)

    try:
        results = sparql.query().convert()
    except QueryBadFormed:
        print "SPARQL query bad formed: " + query_string_with_offset
        return None
    except urllib2.HTTPError:
        print "HTTP Error 502: " + article
        return None
    except urllib2.URLError:
        print "Network is unreachable while working on: " + article
        return None
    except socket.timeout:
        print "Query timed out for: " + article
        return None

    if len(results["results"]["bindings"]) > 0:
        coordinates_array = []
        for result in results["results"]["bindings"]:
            if  "lat" in result:
                latitude = result["lat"]["value"]
                longitude = result["long"]["value"]
            else:
                lat_dir=0
                if "latDir" in result:
                    lat_dir = result["latDir"]["value"]

                latitude = dms2dd(result["latDeg"]["value"], result["latMin"]["value"], result["latSec"]["value"], lat_dir)
                longitude = dms2dd(result["lonDeg"]["value"], result["lonMin"]["value"], result["lonSec"]["value"], lat_dir)

            latitude = float(latitude)
            longitude = float(longitude)
            coordinates_array.append((latitude,longitude))

        majority_vote =  majority_voting.vote(coordinates_array,threshold)
        location = country_lookup.get_country(majority_vote[0],majority_vote[1])

    return location

lang_dump = load_simple_json(inputfile_path)

article_count = 0
# Main loop
for article in lang_dump:
    article_with_underscores = article.replace(' ', '_')
    location=query_location(article_with_underscores)
    if location is not None:
        # print article + " --> " +location
        article_language_dictionary[article]=location

    article_count += 1

    # print article_count

with open(outputfile_path, 'w') as f:
    json.dump(article_language_dictionary, f, indent=4, sort_keys=True)
    print "File was stored successfully"<|endoftext|>


<commit_before>'''Wylie2Uni
    Wylie to Unicode convertor'''

# TODO: Write unit testWylies!

import sys
import tables
from sys import argv

TSHEG = u'\u0f0b'
SUBOFFSET = 0x50

class Translator(object):
    '''Translates wylie into Tibetan Unicode'''

    def __init__(self):
        self.syllable = Syllable('')
        wTable = tables.W_ROOTLETTERS + tables.W_VOWELS
        uTable = tables.U_ROOTLETTERS + tables.U_VOWELS
        self.lookup = dict(zip(wTable, uTable))
        self.rulesSuper = dict(zip(tables.SUPER, tables.SUPER_RULES))
        self.rulesSub = dict(zip(tables.SUB,   tables.SUB_RULES))

    def newSyllable(self):
        self.syllable = Syllable('')

    def toUnicode(self, syllable):
        return self.lookup[str(syllable)]

    def toSubjoinedUnicode(self, syllable):
        return chr(ord(self.lookup[str(syllable)]) + SUBOFFSET)

    def setStruct(self, key, value):
        self.syllable.struct[key] = value

    def clearSyllable(self):
        self.syllable.clear()

    def getCharacterOf(self, key):
        return self.syllable.struct[key]

    def getVowelIndex(self, parts):
        for i, char in enumerate(parts):
            if char in tables.W_VOWELS:
                return i
        return -1

    def isSuperscribed(self, parts, vowelPosition):
        if vowelPosition == 2:
            return self.validSuperscribe(parts[0], parts[1]) \
                and not self.validSubscribe(parts[1], parts[0])
        else:   # vowelPosition == 3
            return self.isPrefix(parts[0]) \
                and self.validSuperscribe(parts[1], parts[2]) \
                and not self.validSubscribe(parts[2], parts[1])

    def isSubscribed(self, parts, vowelPosition):
        if vowelPosition == 2:
            return not self.validSuperscribe(parts[0], parts[1]) \
                and self.validSubscribe(parts[1], parts[0])

        else:   # vowelPosition == 3
            return self.isPrefix(parts[0]) \
                and not self.validSuperscribe(parts[1], parts[2]) \
                and self.validSubscribe(parts[2], parts[1])

    def normalWylie(self, parts, vowelPosition):
        if vowelPosition == 0:
            if parts[0] in tables.W_VOWELS:
                if parts[0] != 'a':
    # TODO This gives incorrect wylie. Move to renderUnicode().
                    self.syllable.wylie = ''.join(['a', parts[0]])
                    self.setStruct('root', 'a')
                    self.setStruct('vowel', parts[0])
                else:
                    self.setStruct('root',  parts[0])
                    self.setStruct('vowel', parts[0])

        if vowelPosition == 1:
            self.setStruct('root', parts[0])
            self.setStruct('vowel', parts[1])

        elif vowelPosition == 2:
            if self.isSubscribed(parts, vowelPosition):
                self.setStruct('root',      parts[0])
                self.setStruct('subjoined', parts[1])

            elif self.isSuperscribed(parts, vowelPosition):
                self.setStruct('super', parts[0])
                self.setStruct('root',  parts[1])

            else:
                self.setStruct('prefix', parts[0])
                self.setStruct('root',   parts[1])

            self.setStruct('vowel',  parts[2])

        elif vowelPosition == 3:
            if self.isIrregular(vowelPosition, parts):
                self.setStruct('root',      parts[0])
                self.setStruct('subjoined', parts[1])
                self.setStruct('secondsub', parts[2])

            elif self.isSuperscribed(parts, vowelPosition):
                self.setStruct('prefix', parts[0])
                self.setStruct('super',  parts[1])
                self.setStruct('root',   parts[2])

            elif self.isSubscribed(parts, vowelPosition):
                self.setStruct('prefix',    parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])

            else:
                self.setStruct('super',     parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])

            self.setStruct('vowel',  parts[3])

        elif vowelPosition == 4:  # (max allowed)
            if self.isIrregular(vowelPosition, parts):
                self.setStruct('prefix',    parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])
                self.setStruct('secondsub', parts[3])
            else:
                self.setStruct('prefix',    parts[0])
                self.setStruct('super',     parts[1])
                self.setStruct('root',      parts[2])
                self.setStruct('subjoined', parts[3])

            self.setStruct('vowel', parts[4])

    def singleWylieCharacter(self, parts):
        self.setStruct('root', parts[0])

        if parts[0] in tables.W_VOWELS:
            self.setStruct('vowel', parts[0])
        else:
            self.setStruct('root', 'a')
            self.setStruct('vowel', parts[0])

    def determineWylie(self):
        parts = self.partition()
        self.clearSyllable()

        if len(parts) == 1:
            self.singleWylieCharacter(parts)

    # TODO Remove redundant vowel check
        if self.noVowels(parts):
            return

        else:
            vowelPosition = self.getVowelIndex(parts)
            self.normalWylie(parts, vowelPosition)
            self.findSuffixes(vowelPosition, parts)

    def isIrregular(self, vowelPosition, parts):
        '''Checks if the syllable has both 'w' and 'r' as subscribed letters'''

        if parts[vowelPosition-1] == 'w' \
                and parts[vowelPosition-2] == 'r':
            return True
        else:
            return False

    def findSuffixes(self, vowelPosition, parts):
    # TODO: Iterator for POSTVOWEL.next *if* that doesn't impede performance
        j = 0
        for i in range(vowelPosition+1, len(parts)):
            wyChar = parts[i]
            if wyChar:
                self.setStruct(tables.POSTVOWEL[j], wyChar)
                j += 1

    def noVowels(self, parts):
        for char in parts:
            if char in tables.W_VOWELS:
                return False

        return True

    def partition(self):
        '''Generates a list of wylie characters, from which the wylie string
            is composed of.'''

        result = []
        syll = self.syllable.wylie

        while len(syll) != 0:
            for i in range(3, 0, -1):
                part = syll[:i]

                if part in tables.W_ROOTLETTERS + tables.W_VOWELS \
                        or part == 'g.':
                    result.append(part)
                    syll = syll[len(part):]

        return result

    def validSuperscribe(self, head, root):
        if head not in tables.SUPER:
            return False
        else:
            return root in self.rulesSuper[head]

    def validSubscribe(self, sub, root):
        if sub not in tables.SUB:
            return False

        else:
            return root in self.rulesSub[sub]

    def appendWylieChar(self, char):
        self.syllable.wylie = ''.join([self.syllable.wylie, char])

    def add(self, char):
        self.appendWylieChar(char)
        self.determineWylie()
        self.renderUnicode()

    def needsSubjoin(self, comp):
        if comp == 'subjoined' or comp == 'secondsub' \
                or (comp == 'root' and self.getCharacterOf('super')):
            return True
        else:
            return False

    def renderUnicode(self):
        for comp in tables.SYLLSTRUCT:
            char = self.getCharacterOf(comp)

            if char == 'a' and comp != 'root':
                continue

            newString = [self.syllable.uni]

            if char:
    # TODO: More efficient handling of g. irregular.
                if char == 'g.':
                    char = 'g'
                    self.setStruct(comp, char)

                if self.needsSubjoin(comp):
                    newString.append(self.toSubjoinedUnicode(char))

                else:
                    newString.append(self.toUnicode(char))

                self.syllable.uni = u''.join(newString)

    def isPrefix(self, char):
        if char in tables.PREFIXES:
            return True
        else:
            return False

    def tsheg(self):
        self.syllable.tsheg()
        return self.syllable.uni

    def alphabet(self):  # {{{
        for i, key in enumerate(tables.W_ROOTLETTERS):
            self.newSyllable()
            self.tsheg()

            if i % 4 == 0:
                sys.stdout.write("\n")

        sys.stdout.write("\n")

    def vowels(self):
        for key in tables.W_VOWELS:
            self.newSyllable('a')
            self.add(key)
            self.tsheg()
        print()

    def testWylie(self, string):
        self.newSyllable()

        for i, s in enumerate(string):
            self.add(s)

        print(string + " : " + self.tsheg())

    # }}}

class Syllable(object):
    def __init__(self, wylie):
        self.wylie = wylie
        self.uni = u''
        self.struct = dict((key, '') for key in tables.SYLLSTRUCT)

    def __str__(self):
        return self.wylie

    def __repr__(self):
        return self.wylie

    def tsheg(self):
        self.uni = u''.join([self.uni, TSHEG])

    def add(self, uni):
        self.uni = u''.join([self.uni, uni])

    def clear(self):
        self.uni = u''
        for s in tables.SYLLSTRUCT:
            self.struct[s] = ''

def main():
    t = Translator()
    # t.alphabet()
    # t.vowels()

    if len(argv) < 2:
        t.testWylie('sangs')
        t.testWylie('bre')
        t.testWylie('rta')
        t.testWylie('mgo')
        t.testWylie('gya')
        t.testWylie('g.yag')
        t.testWylie('\'rba')
        t.testWylie('tshos')
        t.testWylie('lhongs')
        t.testWylie('mngar')
        t.testWylie('sngas')
        t.testWylie('rnyongs')
        t.testWylie('brnyes')
        t.testWylie('rgyas')
        t.testWylie('skyongs')
        t.testWylie('bskyongs')
        t.testWylie('grwa')
        t.testWylie('spre\'u')
        t.testWylie('spre\'u\'i')
        t.testWylie('\'dra')
        t.testWylie('\'bya')
        t.testWylie('\'gra')
        t.testWylie('\'gyang')
        t.testWylie('\'khra')
        t.testWylie('\'khyig')
        t.testWylie('\'kyags')
        t.testWylie('\'phre')
        t.testWylie('\'phyags')
        t.testWylie('a')
        t.testWylie('o')
        t.testWylie('a\'am')
        t.testWylie('ab')
        t.testWylie('bswa')
        # t.testWylie('bha')
        return

    f = open(argv[1], 'r')

    for line in f:
        t.testWylie(line.rstrip())

    f.close()

if __name__ == '__main__':
    main()
<commit_msg>Added printBytecodes().<commit_after>'''Wylie2Uni
    Wylie to Unicode convertor'''

# TODO: Write unit testWylies!

import sys
import tables
from sys import argv

TSHEG = u'\u0f0b'
SUBOFFSET = 0x50

class Translator(object):
    '''Translates wylie into Tibetan Unicode'''

    def __init__(self):
        self.syllable = Syllable('')
        wTable = tables.W_ROOTLETTERS + tables.W_VOWELS
        uTable = tables.U_ROOTLETTERS + tables.U_VOWELS
        self.lookup = dict(zip(wTable, uTable))
        self.rulesSuper = dict(zip(tables.SUPER, tables.SUPER_RULES))
        self.rulesSub = dict(zip(tables.SUB,   tables.SUB_RULES))

    def newSyllable(self):
        self.syllable = Syllable('')

    def toUnicode(self, syllable):
        return self.lookup[str(syllable)]

    def toSubjoinedUnicode(self, syllable):
        return chr(ord(self.lookup[str(syllable)]) + SUBOFFSET)

    def setStruct(self, key, value):
        self.syllable.struct[key] = value

    def clearSyllable(self):
        self.syllable.clear()

    def getCharacterOf(self, key):
        return self.syllable.struct[key]

    def getVowelIndex(self, parts):
        for i, char in enumerate(parts):
            if char in tables.W_VOWELS:
                return i
        return -1

    def isSuperscribed(self, parts, vowelPosition):
        if vowelPosition == 2:
            return self.validSuperscribe(parts[0], parts[1]) \
                and not self.validSubscribe(parts[1], parts[0])
        else:   # vowelPosition == 3
            return self.isPrefix(parts[0]) \
                and self.validSuperscribe(parts[1], parts[2]) \
                and not self.validSubscribe(parts[2], parts[1])

    def isSubscribed(self, parts, vowelPosition):
        if vowelPosition == 2:
            return not self.validSuperscribe(parts[0], parts[1]) \
                and self.validSubscribe(parts[1], parts[0])

        else:   # vowelPosition == 3
            return self.isPrefix(parts[0]) \
                and not self.validSuperscribe(parts[1], parts[2]) \
                and self.validSubscribe(parts[2], parts[1])

    def normalWylie(self, parts, vowelPosition):
        if vowelPosition == 0:
            if parts[0] in tables.W_VOWELS:
                if parts[0] != 'a':
    # TODO This gives incorrect wylie. Move to renderUnicode().
                    self.syllable.wylie = ''.join(['a', parts[0]])
                    self.setStruct('root', 'a')
                    self.setStruct('vowel', parts[0])
                else:
                    self.setStruct('root',  parts[0])
                    self.setStruct('vowel', parts[0])

        if vowelPosition == 1:
            self.setStruct('root', parts[0])
            self.setStruct('vowel', parts[1])

        elif vowelPosition == 2:
            if self.isSubscribed(parts, vowelPosition):
                self.setStruct('root',      parts[0])
                self.setStruct('subjoined', parts[1])

            elif self.isSuperscribed(parts, vowelPosition):
                self.setStruct('super', parts[0])
                self.setStruct('root',  parts[1])

            else:
                self.setStruct('prefix', parts[0])
                self.setStruct('root',   parts[1])

            self.setStruct('vowel',  parts[2])

        elif vowelPosition == 3:
            if self.isIrregular(vowelPosition, parts):
                self.setStruct('root',      parts[0])
                self.setStruct('subjoined', parts[1])
                self.setStruct('secondsub', parts[2])

            elif self.isSuperscribed(parts, vowelPosition):
                self.setStruct('prefix', parts[0])
                self.setStruct('super',  parts[1])
                self.setStruct('root',   parts[2])

            elif self.isSubscribed(parts, vowelPosition):
                self.setStruct('prefix',    parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])

            else:
                self.setStruct('super',     parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])

            self.setStruct('vowel',  parts[3])

        elif vowelPosition == 4:  # (max allowed)
            if self.isIrregular(vowelPosition, parts):
                self.setStruct('prefix',    parts[0])
                self.setStruct('root',      parts[1])
                self.setStruct('subjoined', parts[2])
                self.setStruct('secondsub', parts[3])
            else:
                self.setStruct('prefix',    parts[0])
                self.setStruct('super',     parts[1])
                self.setStruct('root',      parts[2])
                self.setStruct('subjoined', parts[3])

            self.setStruct('vowel', parts[4])

    def singleWylieCharacter(self, parts):
        self.setStruct('root', parts[0])

        if parts[0] in tables.W_VOWELS:
            self.setStruct('vowel', parts[0])
        else:
            self.setStruct('root', 'a')
            self.setStruct('vowel', parts[0])

    def determineWylie(self):
        parts = self.partition()
        self.clearSyllable()

        if len(parts) == 1:
            self.singleWylieCharacter(parts)

    # TODO Remove redundant vowel check
        if self.noVowels(parts):
            return

        else:
            vowelPosition = self.getVowelIndex(parts)
            self.normalWylie(parts, vowelPosition)
            self.findSuffixes(vowelPosition, parts)

    def isIrregular(self, vowelPosition, parts):
        '''Checks if the syllable has both 'w' and 'r' as subscribed letters'''

        if parts[vowelPosition-1] == 'w' \
                and parts[vowelPosition-2] == 'r':
            return True
        else:
            return False

    def findSuffixes(self, vowelPosition, parts):
    # TODO: Iterator for POSTVOWEL.next *if* that doesn't impede performance
        j = 0
        for i in range(vowelPosition+1, len(parts)):
            wyChar = parts[i]
            if wyChar:
                self.setStruct(tables.POSTVOWEL[j], wyChar)
                j += 1

    def noVowels(self, parts):
        for char in parts:
            if char in tables.W_VOWELS:
                return False

        return True

    def partition(self):
        '''Generates a list of wylie characters, from which the wylie string
            is composed of.'''

        result = []
        syll = self.syllable.wylie

        while len(syll) != 0:
            for i in range(3, 0, -1):
                part = syll[:i]

                if part in tables.W_ROOTLETTERS + tables.W_VOWELS \
                        or part == 'g.':
                    result.append(part)
                    syll = syll[len(part):]

        return result

    def validSuperscribe(self, head, root):
        if head not in tables.SUPER:
            return False
        else:
            return root in self.rulesSuper[head]

    def validSubscribe(self, sub, root):
        if sub not in tables.SUB:
            return False

        else:
            return root in self.rulesSub[sub]

    def appendWylieChar(self, char):
        self.syllable.wylie = ''.join([self.syllable.wylie, char])

    def add(self, char):
        self.appendWylieChar(char)
        self.determineWylie()
        self.renderUnicode()

    def needsSubjoin(self, comp):
        if comp == 'subjoined' or comp == 'secondsub' \
                or (comp == 'root' and self.getCharacterOf('super')):
            return True
        else:
            return False

    def renderUnicode(self):
        for comp in tables.SYLLSTRUCT:
            char = self.getCharacterOf(comp)

            if char == 'a' and comp != 'root':
                continue

            newString = [self.syllable.uni]

            if char:
    # TODO: More efficient handling of g. irregular.
                if char == 'g.':
                    char = 'g'
                    self.setStruct(comp, char)

                if self.needsSubjoin(comp):
                    newString.append(self.toSubjoinedUnicode(char))

                else:
                    newString.append(self.toUnicode(char))

                self.syllable.uni = u''.join(newString)

    def isPrefix(self, char):
        if char in tables.PREFIXES:
            return True
        else:
            return False

    def tsheg(self):
        self.syllable.tsheg()
        return self.syllable.uni

    def alphabet(self):  # {{{
        for i, key in enumerate(tables.W_ROOTLETTERS):
            self.newSyllable()
            self.tsheg()

            if i % 4 == 0:
                sys.stdout.write("\n")

        sys.stdout.write("\n")

    def vowels(self):
        for key in tables.W_VOWELS:
            self.newSyllable('a')
            self.add(key)
            self.tsheg()
        print()

    def testWylie(self, string):
        self.newSyllable()

        for i, s in enumerate(string):
            self.add(s)

        print(string + " : " + self.tsheg())

    def printBytecodes(self, string):
        self.newSyllable()

        for i, s in enumerate(string):
            self.add(s)

        for uni in self.syllable.uni:
            print('U+{0:04X}'.format(ord(uni)))

    # }}}

class Syllable(object):
    def __init__(self, wylie):
        self.wylie = wylie
        self.uni = u''
        self.struct = dict((key, '') for key in tables.SYLLSTRUCT)

    def __str__(self):
        return self.wylie

    def __repr__(self):
        return self.wylie

    def tsheg(self):
        self.uni = u''.join([self.uni, TSHEG])

    def add(self, uni):
        self.uni = u''.join([self.uni, uni])

    def clear(self):
        self.uni = u''
        for s in tables.SYLLSTRUCT:
            self.struct[s] = ''

def main():
    t = Translator()
    # t.alphabet()
    # t.vowels()

    if len(argv) < 2:
        t.printBytecodes('bskyongs')
        # t.testWylie('sangs')
        # t.testWylie('bre')
        # t.testWylie('rta')
        # t.testWylie('mgo')
        # t.testWylie('gya')
        # t.testWylie('g.yag')
        # t.testWylie('\'rba')
        # t.testWylie('tshos')
        # t.testWylie('lhongs')
        # t.testWylie('mngar')
        # t.testWylie('sngas')
        # t.testWylie('rnyongs')
        # t.testWylie('brnyes')
        # t.testWylie('rgyas')
        # t.testWylie('skyongs')
        # t.testWylie('bskyongs')
        # t.testWylie('grwa')
        # t.testWylie('spre\'u')
        # t.testWylie('spre\'u\'i')
        # t.testWylie('\'dra')
        # t.testWylie('\'bya')
        # t.testWylie('\'gra')
        # t.testWylie('\'gyang')
        # t.testWylie('\'khra')
        # t.testWylie('\'khyig')
        # t.testWylie('\'kyags')
        # t.testWylie('\'phre')
        # t.testWylie('\'phyags')
        # t.testWylie('a')
        # t.testWylie('o')
        # t.testWylie('a\'am')
        # t.testWylie('ab')
        # t.testWylie('bswa')
        # t.testWylie('bha')
        return

    f = open(argv[1], 'r')

    for line in f:
        t.testWylie(line.rstrip())

    f.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import json
import Gnuplot, Gnuplot.PlotItems, Gnuplot.funcutils
import argparse
import time, os, sys
import tempfile
import math
def main():
    parser = argparse.ArgumentParser(description = "Plots and saves json objects in folder parseddata, uses a config file for plot setting inside cfg/.plotjson for gnuplot settings")
    parser.add_argument('dir', metavar = 'd', nargs = '+', help="Timestamp of  json file to be plotted.")
    parser.add_argument('title', metavar = 't', nargs = '+', help = "Title to be included in plot.")
    parser.add_argument('-e', '--errorbars', help = "Use this flag to NOT plot with error bars, in case the plot is too messy.", action = "store_true")

    args = parser.parse_args()
    p = plotJson(args.errorbars)
    p.load(args.dir[0], args.title[0])

def hms(x):
    m, s = divmod(x, 60)
    h, m = divmod(m, 60)
    return "%dh:%02dm:%02ds" % (h, m, s)
class plotJson():
    def __init__(self, eb):
        self.eb = eb
        self.cfg = {}
        with open('cfg/.plotjson', 'rb+') as f:
            x = f.read()
            x = x.split('\n')
            for i in x:
                if len(i) > 0:
                    i = i.rstrip()
                    i = i.split('=')
                    self.cfg[i[0]] = i[1]
    def getx(self,hist, desc):
        h_offset = float(desc['horiz_offset']) * 10 ** 9
        h_binsize = float(desc['horiz_interval']) * 10 ** 9
        s = []
        for i in xrange(len(hist)):
            s.append([(i * h_binsize) + h_offset, hist[i]])
        return s
    def load(self, path, title):
        fpath = 'parseddata/' + path

        with open(fpath + '.json', 'rb+') as datafile:
            data = json.load(datafile)
        if not isinstance(data['hist'][0], list):
            data['hist'] = self.getx(data['hist'], data['desc'])
        duration = int(float(data['desc']['acq_duration']))
        duration = hms(duration)

        rawf = open(fpath, 'wb+')
        for i in xrange(len(data['hist'])):
            _x = data['hist'][i][0]
            _y = data['hist'][i][1]
            _yerror = round(math.sqrt(_y),1)
            rawf.write("{}\t{}\n".format(_x, _y))
        rawf.close()

        self.initPlot()
        self.g('set title "{} {}, acquisition duration {}"'.format(path,title,duration))
        self.g('set output "{}.eps"'.format(fpath))
        if not self.eb:
            self.g('f(x) = mean_y')
            self.g('fit f(x) "{}" u 1:2 via mean_y'.format(fpath))
            self.g('plot "{}" u 1:2:(sqrt(mean_y)) with yerrorbars pt 7 ps 0.2 '.format(fpath))
        else:
            self.g('plot "{}" u 1:2 w p pt 7 ps 0.2'.format(fpath))

    def initPlot(self):
        self.g = Gnuplot.Gnuplot()
        self.g('set term {} transparent truecolor size 10,7.5'.format(self.cfg['format']))
        self.g('set xlabel "{}"'.format(self.cfg['xlabel']))
        self.g('set ylabel {}'.format(self.cfg['ylabel']))

main()
<commit_msg>edited so that it can take in custom filenames from custom locations<commit_after>import json
import Gnuplot, Gnuplot.PlotItems, Gnuplot.funcutils
import argparse
import time, os, sys
import tempfile
import math

def check_dir(directory):
    if not os.path.exists(directory):
        print "Directory {} does not exist...creating...".format(directory)
        os.makedirs(directory)

def main():
    parser = argparse.ArgumentParser(description = "Plots and saves json objects in folder parseddata, uses a config file for plot setting inside cfg/.plotjson for gnuplot settings")
    parser.add_argument('f', metavar = 'f', help="Filename of json file to be plotted.")
    parser.add_argument('title', metavar = 't', help = "Title to be included in plot.")
    parser.add_argument('-e', '--errorbars', help = "Use this flag to NOT plot with error bars, in case the plot is too messy.", action = "store_true")

    args = parser.parse_args()
    p = plotJson(args.errorbars)
    p.load(args.f, args.title)

def hms(x):
    m, s = divmod(x, 60)
    h, m = divmod(m, 60)
    return "%dh:%02dm:%02ds" % (h, m, s)
class plotJson():
    def __init__(self, eb):
        self.eb = eb
        self.cfg = {}
        try:
            with open('cfg/.plotjson', 'rb+') as f:
                x = f.read()
                x = x.split('\n')
                for i in x:
                    if len(i) > 0:
                        i = i.rstrip()
                        i = i.split('=')
                        self.cfg[i[0]] = i[1]
        except:
            print "No config file found...using default settings";
            self.cfg =
            {
                "format": "epscairo",
                "xlabel": "t@level (ns)",
                "ylabel": "Two-photon coincidence events"
            }
    def getx(self,hist, desc):
        h_offset = float(desc['horiz_offset']) * 10 ** 9
        h_binsize = float(desc['horiz_interval']) * 10 ** 9
        s = []
        for i in xrange(len(hist)):
            s.append([(i * h_binsize) + h_offset, hist[i]])
        return s
    def load(self, path, title):
        #fpath = 'parseddata/' + path
        fpath = path
        with open(fpath, 'rb+') as datafile:
            data = json.load(datafile)
        if not isinstance(data['hist'][0], list):
            data['hist'] = self.getx(data['hist'], data['desc'])
        duration = int(float(data['desc']['acq_duration']))
        duration = hms(duration)

        rawf = open(fpath + ".dat", 'wb+')
        for i in xrange(len(data['hist'])):
            _x = data['hist'][i][0]
            _y = data['hist'][i][1]
            _yerror = round(math.sqrt(_y),1)
            rawf.write("{}\t{}\n".format(_x, _y))
        rawf.close()

        self.initPlot()
        self.g('set title "{} {}, acquisition duration {}"'.format(path,title,duration))
        self.g('set output "{}.eps"'.format(fpath))
        if not self.eb:
            self.g('f(x) = mean_y')
            self.g('fit f(x) "{}" u 1:2 via mean_y'.format(fpath))
            self.g('plot "{}" u 1:2:(sqrt(mean_y)) with yerrorbars pt 7 ps 0.2 '.format(fpath))
        else:
            self.g('plot "{}" u 1:2 w p pt 7 ps 0.2'.format(fpath))

    def initPlot(self):
        self.g = Gnuplot.Gnuplot()
        self.g('set term {} transparent truecolor size 10,7.5'.format(self.cfg['format']))
        self.g('set xlabel "{}"'.format(self.cfg['xlabel']))
        self.g('set ylabel {}'.format(self.cfg['ylabel']))

main()
<|endoftext|>