<commit_before>import redis
from casexml.apps.case.exceptions import IllegalCaseId
from corehq.form_processor.backends.sql.update_strategy import SqlCaseUpdateStrategy
from corehq.form_processor.casedb_base import AbstractCaseDbCache
from corehq.form_processor.models import CommCareCaseSQL

class CaseDbCacheSQL(AbstractCaseDbCache):
    case_model_classes = (CommCareCaseSQL,)
    case_update_strategy = SqlCaseUpdateStrategy

    def __init__(self, domain=None, strip_history=False, deleted_ok=False,
                 lock=False, wrap=True, initial=None, xforms=None):
        super(CaseDbCacheSQL, self).__init__(domain, strip_history, deleted_ok, lock, wrap, initial, xforms)
        if not self.wrap:
            raise ValueError('CaseDbCacheSQL does not support unwrapped models')

    def _validate_case(self, case):
        if self.domain and case.domain != self.domain:
            raise IllegalCaseId("Bad case id")
        elif case.is_deleted:
            if not self.deleted_ok:
                raise IllegalCaseId("Case [%s] is deleted " % case.case_id)

    def _get_case(self, case_id):
        try:
            if self.lock:
                try:
                    case, lock = CommCareCaseSQL.get_locked_obj(_id=case_id)
                except redis.RedisError:
                    case = CommCareCaseSQL.get(case_id)
                else:
                    self.locks.append(lock)
            else:
                case = CommCareCaseSQL.get(case_id)
        except CommCareCaseSQL.DoesNotExist:
            return None

        return case

    def _iter_cases(self, case_ids):
        return CommCareCaseSQL.objects.filter(case_uuid__in=case_ids).all()

    def get_cases_for_saving(self, now):
        cases = self.get_changed()

        for case in cases:
            if case.is_saved():
                rev = CommCareCaseSQL.objects.filter(
                    case_uuid=case.case_id,
                    server_modified_on=case.server_modified_on
                )
                assert rev.exists(), (
                    "Aborting because there would have been "
                    "a document update conflict. {}".format(case.case_id)
                )
            case.server_modified_on = now
        return cases

    def get_reverse_indexed_cases(self, case_ids):
        return CommCareCaseSQL.objects.filter(
            domain=self.domain, index__referenced_id__in=case_ids
        ).defer("case_json").prefetch_related('indices  ')
<commit_msg>change 'resource conflict' wording<commit_after>import redis
from casexml.apps.case.exceptions import IllegalCaseId
from corehq.form_processor.backends.sql.update_strategy import SqlCaseUpdateStrategy
from corehq.form_processor.casedb_base import AbstractCaseDbCache
from corehq.form_processor.models import CommCareCaseSQL

class CaseDbCacheSQL(AbstractCaseDbCache):
    case_model_classes = (CommCareCaseSQL,)
    case_update_strategy = SqlCaseUpdateStrategy

    def __init__(self, domain=None, strip_history=False, deleted_ok=False,
                 lock=False, wrap=True, initial=None, xforms=None):
        super(CaseDbCacheSQL, self).__init__(domain, strip_history, deleted_ok, lock, wrap, initial, xforms)
        if not self.wrap:
            raise ValueError('CaseDbCacheSQL does not support unwrapped models')

    def _validate_case(self, case):
        if self.domain and case.domain != self.domain:
            raise IllegalCaseId("Bad case id")
        elif case.is_deleted:
            if not self.deleted_ok:
                raise IllegalCaseId("Case [%s] is deleted " % case.case_id)

    def _get_case(self, case_id):
        try:
            if self.lock:
                try:
                    case, lock = CommCareCaseSQL.get_locked_obj(_id=case_id)
                except redis.RedisError:
                    case = CommCareCaseSQL.get(case_id)
                else:
                    self.locks.append(lock)
            else:
                case = CommCareCaseSQL.get(case_id)
        except CommCareCaseSQL.DoesNotExist:
            return None

        return case

    def _iter_cases(self, case_ids):
        return CommCareCaseSQL.objects.filter(case_uuid__in=case_ids).all()

    def get_cases_for_saving(self, now):
        cases = self.get_changed()

        for case in cases:
            if case.is_saved():
                unchanged_case = CommCareCaseSQL.objects.filter(
                    case_uuid=case.case_id,
                    server_modified_on=case.server_modified_on
                )
                assert unchanged_case.exists(), (
                    "Aborting because the case has been modified"
                    " by another process. {}".format(case.case_id)
                )
            case.server_modified_on = now
        return cases

    def get_reverse_indexed_cases(self, case_ids):
        return CommCareCaseSQL.objects.filter(
            domain=self.domain, index__referenced_id__in=case_ids
        ).defer("case_json").prefetch_related('indices  ')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
# Copyright (C) 2013-Today  Carlos Eduardo Vercelino - CLVsol
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

from __future__ import print_function

import sqlite3

def person_address_role_export_sqlite(client, args, db_path, table_name):

    conn = sqlite3.connect(db_path)
    conn.text_factory = str

    cursor = conn.cursor()
    try:
        cursor.execute('''DROP TABLE ''' + table_name + ''';''')
    except Exception as e:
        print('------->', e)
    cursor.execute('''
        CREATE TABLE ''' + table_name + ''' (
            id INTEGER NOT NULL PRIMARY KEY,
            name,
            code,
            description,
            notes,
            new_id INTEGER
            );
    ''')

    myo_person_address_role = client.model('myo.person.address.role')
    person_address_role_browse = myo_person_address_role.browse(args)

    person_address_role_count = 0
    for person_address_role in person_address_role_browse:
        person_address_role_count += 1

        print(
            person_address_role_count, person_address_role.id, person_address_role.code,
            person_address_role.name.encode("utf-8"), person_address_role.notes
        )

        code = None
        if person_address_role.code:
            code = person_address_role.code

        description = None
        if person_address_role.description:
            description = person_address_role.description

        notes = None
        if person_address_role.notes:
            notes = person_address_role.notes

        cursor.execute('''
                       INSERT INTO ''' + table_name + '''(
                           id,
                           name,
                           code,
                           description,
                           notes
                           )
                       VALUES(?,?,?,?,?)''',
                       (person_address_role.id,
                        person_address_role.name,
                        code,
                        description,
                        notes,
                        )
                       )

    conn.commit()
    conn.close()

    print()
    print('--> person_address_role_count: ', person_address_role_count)
<commit_msg>[IMP] myo_person_address_role.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
# Copyright (C) 2013-Today  Carlos Eduardo Vercelino - CLVsol
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

from __future__ import print_function

import sqlite3

def person_address_role_export_sqlite(client, args, db_path, table_name):

    conn = sqlite3.connect(db_path)
    conn.text_factory = str

    cursor = conn.cursor()
    try:
        cursor.execute('''DROP TABLE ''' + table_name + ''';''')
    except Exception as e:
        print('------->', e)
    cursor.execute('''
        CREATE TABLE ''' + table_name + ''' (
            id INTEGER NOT NULL PRIMARY KEY,
            name,
            code,
            description,
            notes,
            new_id INTEGER
            );
    ''')

    myo_person_address_role = client.model('myo.person.address.role')
    person_address_role_browse = myo_person_address_role.browse(args)

    person_address_role_count = 0
    for person_address_role in person_address_role_browse:
        person_address_role_count += 1

        print(
            person_address_role_count, person_address_role.id, person_address_role.code,
            person_address_role.name.encode("utf-8"), person_address_role.notes
        )

        code = None
        if person_address_role.code:
            code = person_address_role.code

        description = None
        if person_address_role.description:
            description = person_address_role.description

        notes = None
        if person_address_role.notes:
            notes = person_address_role.notes

        cursor.execute('''
                       INSERT INTO ''' + table_name + '''(
                           id,
                           name,
                           code,
                           description,
                           notes
                           )
                       VALUES(?,?,?,?,?)''',
                       (person_address_role.id,
                        person_address_role.name,
                        code,
                        description,
                        notes,
                        )
                       )

    conn.commit()
    conn.close()

    print()
    print('--> person_address_role_count: ', person_address_role_count)

def person_address_role_import_sqlite(client, args, db_path, table_name):

    person_address_role_model = client.model('myo.person.address.role')

    conn = sqlite3.connect(db_path)
    conn.row_factory = sqlite3.Row

    cursor = conn.cursor()

    cursor2 = conn.cursor()

    data = cursor.execute('''
        SELECT
            id,
            name,
            code,
            description,
            notes,
            new_id
        FROM ''' + table_name + ''';
    ''')

    print(data)
    print([field[0] for field in cursor.description])

    person_address_role_count = 0
    for row in cursor:
        person_address_role_count += 1

        print(
            person_address_role_count, row['id'], row['name'], row['code'],
            row['description'], row['notes'],
        )

        values = {
            'name': row['name'],
            'code': row['code'],
            'description': row['description'],
            'notes': row['notes'],
        }
        person_address_role_id = person_address_role_model.create(values).id

        cursor2.execute(
            '''
            UPDATE ''' + table_name + '''
            SET new_id = ?
            WHERE id = ?;''',
            (person_address_role_id,
             row['id']
             )
        )

    conn.commit()
    conn.close()

    print()
    print('--> person_address_role_count: ', person_address_role_count)
<|endoftext|>


<commit_before>import functools
import heapq
import multiprocessing
import re
import signal

class MatchInfo(object):
    def __init__(self, start=0, end=0, score=0, round_ejected=0):
        """
        Information about how a patch matches against the given search

        @attr start         - index in path where match starts
        @attr end           - index in path where match ends
        @attr score         - match score
        @attr round_ejected - the round (length of search string) that this
                              path was removed as a possible match
        """
        self.start = start
        self.end = end
        self.score = score
        self.round_ejected = round_ejected

    def update(self, start=None, end=None, score=None, round_ejected=None):
        """
        Update the MatchInfo attributes

        @param start            - index in path where match starts
        @param end              - index in path where match ends
        @param score            - match score
        @param round_ejected    - the round (length of search string) that this
                                  path
        """
        if start is not None:
            self.start = start
        if end is not None:
            self.end = end
        if score is not None:
            self.score = score
        if round_ejected is not None:
            self.round_ejected = round_ejected

def default_scorer(path, c_round, regex):
    """
    Score how well a path is matched by the given regex.  Note
    that this function will be passed to a multiprocessing pool
    and needs to return enough context such that the results can
    be correctly collated back at the source.

    @param path     - path to score
    @param c_round  - length of the current search query.  This is
                      used to support deletion of characters in the
                      query.  If a character is deleted, then all
                      results that were ejected in the previous round
                      will be re-evaluated
    @param regex    - regular expression to use for matching
    @return         - This is a little complex in order to support
                      multiprocessing.  The return is a tuple where
                      the first item is the path that was passed into
                      this function.  The second item is another tuple
                      consisting of MatchInfo attributes: start, end,
                      score, round_ejected.  That is:
                      (path, (start, end, score, round_ejected))
    """
    matches = [m for m in regex.finditer(path)]
    if matches:

        def score(match):
            return 1.0 / (len(path) - match.start(1))

        ranked = [(score(m), m) for m in matches]
        score, best = max(ranked)

        return path, (best.start(1), best.end(1) - 1, score, 0)
    else:
        return path, (0, 0, 0.0, c_round)

class FuzzyMatch(object):
    def __init__(self, files=None, scorer=default_scorer):
        """
        Create a FuzzyMatcher which is responsible for handling the
        state as paths are added to the library being searched by an
        updating query.

        @param files    - initial library of paths to rank
        @param scorer   - scoring function.  This function must match
                          the signature and return values of the
                          default_scorer.  See it for more information.
        """
        if files is not None:
            self._library = {path: MatchInfo() for path in files}
        else:
            self._library = {}

        self._scorer = scorer
        self._search = ""

        def pool_init():
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        self._pool = multiprocessing.Pool(initializer=pool_init)

    @property
    def n_matches(self):
        """
        Number of paths which are candidates given the current query
        """
        return len([info for info in self._library.values() if info.round_ejected == 0])

    @property
    def n_files(self):
        """
        Total number of paths in the library being searched
        """
        return len(self._library)

    def add_files(self, files):
        """
        Add files to the library being searched.  This does not automatically
        score the new files.  Use update_scores() to do so if necessary.

        @param files    - list of files to add.
        """
        self._library.update({path: MatchInfo() for path in files})

    def reset_files(self, files):
        """
        Reset the library of possible files to match.  This does not
        automatically score the new files.  Use update_scores() to do
        so if necessary.

        @param files    - new files to use as a library
        """
        self._library = {path: MatchInfo() for path in files}

    def update_scores(self, search):
        """
        Update the scores for every path in the library that was not
        previously ejected.  If the search term is smaller than the
        previous one, paths that were ejected in the last round will
        be re-scored and possibly retained.

        @param search   - query to fuzzy match against the library
        """
        s_len = len(search)

        if s_len < len(self._search):
            for path, info in self._library.items():
                if info.round_ejected == s_len + 1:
                    self._library[path].update(round_ejected=0)

        self._search = search

        if s_len == 0:
            return
        elif s_len == 1:
            # In the single character search case, it's far faster to use the
            # string index function over regex.  This is important as we want
            # to slim down the potential matches as quickly as possible.
            def quick_score(path, info):
                index = path.find(search)
                if index != -1:
                    info.update(start=index, end=index, score=1.0)
                else:
                    info.update(round_ejected=1)

            for path, info in self._library.items():
                quick_score(path, info)
            return

        pattern = "(?=(" + ".*?".join(re.escape(c) for c in search) + "))"
        regex = re.compile(pattern, re.IGNORECASE)

        scorer = functools.partial(self._scorer, c_round=s_len, regex=regex)
        candidates = [
            path for path, info in self._library.items() if info.round_ejected == 0
        ]

        for path, update in self._pool.map(scorer, candidates):
            self._library[path].update(*update)

    def score(self, path):
        """
        @param path - path to lookup
        @return     - score of the given path
        """
        return self._library[path].score

    def start(self, path):
        """
        @param path - path to lookup
        @return     - index of the start of the match of the current query in
                      the path
        """
        return self._library[path].start

    def end(self, path):
        """
        @param path - path to lookup
        @return     - index of the end of the match of the current query in the
                      path
        """
        return self._library[path].end

    def top_matches(self, depth=10):
        """
        Get the best matching paths in the library.  Note that only paths which
        have not been ejected and have a positive score will be returned.
        Therefore, the length of the returned list may be less than the
        specified depth.

        @param depth    - maximum number of paths to return
        @return         - sorted list of the top scoring paths in the library
        """
        if len(self._search) > 0:
            valid = [
                path
                for path, info in self._library.items()
                if info.score > 0 and info.round_ejected == 0
            ]
        else:
            valid = self._library.keys()

        ret = heapq.nlargest(depth, valid, key=lambda x: self._library[x].score)
        return ret
<commit_msg>core: close multiprocessing pool when finished<commit_after>import functools
import heapq
import multiprocessing
import re
import signal

class MatchInfo(object):
    def __init__(self, start=0, end=0, score=0, round_ejected=0):
        """
        Information about how a patch matches against the given search

        @attr start         - index in path where match starts
        @attr end           - index in path where match ends
        @attr score         - match score
        @attr round_ejected - the round (length of search string) that this
                              path was removed as a possible match
        """
        self.start = start
        self.end = end
        self.score = score
        self.round_ejected = round_ejected

    def update(self, start=None, end=None, score=None, round_ejected=None):
        """
        Update the MatchInfo attributes

        @param start            - index in path where match starts
        @param end              - index in path where match ends
        @param score            - match score
        @param round_ejected    - the round (length of search string) that this
                                  path
        """
        if start is not None:
            self.start = start
        if end is not None:
            self.end = end
        if score is not None:
            self.score = score
        if round_ejected is not None:
            self.round_ejected = round_ejected

def default_scorer(path, c_round, regex):
    """
    Score how well a path is matched by the given regex.  Note
    that this function will be passed to a multiprocessing pool
    and needs to return enough context such that the results can
    be correctly collated back at the source.

    @param path     - path to score
    @param c_round  - length of the current search query.  This is
                      used to support deletion of characters in the
                      query.  If a character is deleted, then all
                      results that were ejected in the previous round
                      will be re-evaluated
    @param regex    - regular expression to use for matching
    @return         - This is a little complex in order to support
                      multiprocessing.  The return is a tuple where
                      the first item is the path that was passed into
                      this function.  The second item is another tuple
                      consisting of MatchInfo attributes: start, end,
                      score, round_ejected.  That is:
                      (path, (start, end, score, round_ejected))
    """
    matches = [m for m in regex.finditer(path)]
    if matches:

        def score(match):
            return 1.0 / (len(path) - match.start(1))

        ranked = [(score(m), m) for m in matches]
        score, best = max(ranked)

        return path, (best.start(1), best.end(1) - 1, score, 0)
    else:
        return path, (0, 0, 0.0, c_round)

class FuzzyMatch(object):
    def __init__(self, files=None, scorer=default_scorer):
        """
        Create a FuzzyMatcher which is responsible for handling the
        state as paths are added to the library being searched by an
        updating query.

        @param files    - initial library of paths to rank
        @param scorer   - scoring function.  This function must match
                          the signature and return values of the
                          default_scorer.  See it for more information.
        """
        if files is not None:
            self._library = {path: MatchInfo() for path in files}
        else:
            self._library = {}

        self._scorer = scorer
        self._search = ""

        def pool_init():
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        self._pool = multiprocessing.Pool(initializer=pool_init)

    def __del__(self):
        self._pool.close()

    @property
    def n_matches(self):
        """
        Number of paths which are candidates given the current query
        """
        return len([info for info in self._library.values() if info.round_ejected == 0])

    @property
    def n_files(self):
        """
        Total number of paths in the library being searched
        """
        return len(self._library)

    def add_files(self, files):
        """
        Add files to the library being searched.  This does not automatically
        score the new files.  Use update_scores() to do so if necessary.

        @param files    - list of files to add.
        """
        self._library.update({path: MatchInfo() for path in files})

    def reset_files(self, files):
        """
        Reset the library of possible files to match.  This does not
        automatically score the new files.  Use update_scores() to do
        so if necessary.

        @param files    - new files to use as a library
        """
        self._library = {path: MatchInfo() for path in files}

    def update_scores(self, search):
        """
        Update the scores for every path in the library that was not
        previously ejected.  If the search term is smaller than the
        previous one, paths that were ejected in the last round will
        be re-scored and possibly retained.

        @param search   - query to fuzzy match against the library
        """
        s_len = len(search)

        if s_len < len(self._search):
            for path, info in self._library.items():
                if info.round_ejected == s_len + 1:
                    self._library[path].update(round_ejected=0)

        self._search = search

        if s_len == 0:
            return
        elif s_len == 1:
            # In the single character search case, it's far faster to use the
            # string index function over regex.  This is important as we want
            # to slim down the potential matches as quickly as possible.
            def quick_score(path, info):
                index = path.find(search)
                if index != -1:
                    info.update(start=index, end=index, score=1.0)
                else:
                    info.update(round_ejected=1)

            for path, info in self._library.items():
                quick_score(path, info)
            return

        pattern = "(?=(" + ".*?".join(re.escape(c) for c in search) + "))"
        regex = re.compile(pattern, re.IGNORECASE)

        scorer = functools.partial(self._scorer, c_round=s_len, regex=regex)
        candidates = [
            path for path, info in self._library.items() if info.round_ejected == 0
        ]

        for path, update in self._pool.map(scorer, candidates):
            self._library[path].update(*update)

    def score(self, path):
        """
        @param path - path to lookup
        @return     - score of the given path
        """
        return self._library[path].score

    def start(self, path):
        """
        @param path - path to lookup
        @return     - index of the start of the match of the current query in
                      the path
        """
        return self._library[path].start

    def end(self, path):
        """
        @param path - path to lookup
        @return     - index of the end of the match of the current query in the
                      path
        """
        return self._library[path].end

    def top_matches(self, depth=10):
        """
        Get the best matching paths in the library.  Note that only paths which
        have not been ejected and have a positive score will be returned.
        Therefore, the length of the returned list may be less than the
        specified depth.

        @param depth    - maximum number of paths to return
        @return         - sorted list of the top scoring paths in the library
        """
        if len(self._search) > 0:
            valid = [
                path
                for path, info in self._library.items()
                if info.score > 0 and info.round_ejected == 0
            ]
        else:
            valid = self._library.keys()

        ret = heapq.nlargest(depth, valid, key=lambda x: self._library[x].score)
        return ret
<|endoftext|>


<commit_before>import logging
from datetime import datetime

from django.contrib.auth.models import User
from django.core import paginator
from django.dispatch.dispatcher import receiver
from django.contrib import messages
from django.core.paginator import Paginator
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
from django.views.decorators.http import require_http_methods
from django.contrib.auth.decorators import login_required

from ..text import escape_user_input, misencode
from ..models import UserProfile, Letter

FORM_DELETE = 1
FORM_SEND = 2
FORM_REPLY = 4

DEFAULT_LIMIT = 10
DEFAULT_PAGE = 1

logger = logging.getLogger(__name__)

@login_required
@require_http_methods(["HEAD", "GET", "POST"])
def postal_service(request):
    if request.method == "POST":
        return handle_postal_service_post_request(request)

    nick = request.user.profile.nick
    per_page = int(request.GET.get("l", DEFAULT_LIMIT))
    page = int(request.GET.get("z_s", DEFAULT_PAGE))

    letters = Letter.objects.filter(
        ((Q(receiver=nick) | Q(sender=nick)) & Q(visibility=3))
        | (Q(receiver=nick) & Q(visibility=2))
        | (Q(sender=nick) & Q(visibility=1))
    ).order_by("-date")

    box_occupancy = Letter.objects.filter(
        (Q(receiver=nick) | Q(sender=nick)) & Q(visibility=1)
    ).count()

    paginator = Paginator(letters, per_page)
    letters = paginator.get_page(page)

    return render(
        request,
        "postal_service/postal_office.html",
        {
            "reply_id": FORM_REPLY,
            "send_id": FORM_SEND,
            "delete_id": FORM_DELETE,
            "letters": letters,
            "per_page": per_page if per_page != DEFAULT_LIMIT else False,
            "box_occupancy": box_occupancy,
        },
    )

def handle_postal_service_post_request(request):
    fid = int(request.POST.get("fid"))
    if fid == FORM_SEND:
        try:
            Letter.objects.create(
                receiver=UserProfile.objects.get(
                    nick=misencode(request.POST.get("whom"))
                ).nick,
                sender=request.user.userprofile.nick,
                text=request.POST.get("text"),
                date=datetime.now(),
                visibility=1,
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
        except UserProfile.DoesNotExist:
            id = request.POST.get("whom")
            logger.error(
                f"A message between user has been submitted but the receiver of userprofile can not be found in our database. ID: {id}"
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))

    elif fid == FORM_DELETE:

        try:
            letter = Letter.objects.filter(pk=request.POST.get("id", 0)).update(
                visibility=0
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
        except Letter.DoesNotExist:
            id = request.POST.get("id")
            user = request.user.userprofile.nick
            logger.error(
                f"There has been an attempt to delete a message with non existing id: {id}, User: {user}"
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
<commit_msg>fix: Visibility from 1 to 3 when new message is send.<commit_after>import logging
from datetime import datetime

from django.contrib.auth.models import User
from django.core import paginator
from django.dispatch.dispatcher import receiver
from django.contrib import messages
from django.core.paginator import Paginator
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
from django.views.decorators.http import require_http_methods
from django.contrib.auth.decorators import login_required

from ..text import escape_user_input, misencode
from ..models import UserProfile, Letter

FORM_DELETE = 1
FORM_SEND = 2
FORM_REPLY = 4

DEFAULT_LIMIT = 10
DEFAULT_PAGE = 1

logger = logging.getLogger(__name__)

@login_required
@require_http_methods(["HEAD", "GET", "POST"])
def postal_service(request):
    if request.method == "POST":
        return handle_postal_service_post_request(request)

    nick = request.user.profile.nick
    per_page = int(request.GET.get("l", DEFAULT_LIMIT))
    page = int(request.GET.get("z_s", DEFAULT_PAGE))

    letters = Letter.objects.filter(
        ((Q(receiver=nick) | Q(sender=nick)) & Q(visibility=3))
        | (Q(receiver=nick) & Q(visibility=2))
        | (Q(sender=nick) & Q(visibility=1))
    ).order_by("-date")

    box_occupancy = Letter.objects.filter(
        (Q(receiver=nick) | Q(sender=nick)) & Q(visibility=1)
    ).count()

    paginator = Paginator(letters, per_page)
    letters = paginator.get_page(page)

    return render(
        request,
        "postal_service/postal_office.html",
        {
            "reply_id": FORM_REPLY,
            "send_id": FORM_SEND,
            "delete_id": FORM_DELETE,
            "letters": letters,
            "per_page": per_page if per_page != DEFAULT_LIMIT else False,
            "box_occupancy": box_occupancy,
        },
    )

def handle_postal_service_post_request(request):
    fid = int(request.POST.get("fid"))
    if fid == FORM_SEND:
        try:
            Letter.objects.create(
                receiver=UserProfile.objects.get(
                    nick=misencode(request.POST.get("whom"))
                ).nick,
                sender=request.user.userprofile.nick,
                text=request.POST.get("text"),
                date=datetime.now(),
                visibility=3,
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
        except UserProfile.DoesNotExist:
            id = request.POST.get("whom")
            logger.error(
                f"A message between user has been submitted but the receiver of userprofile can not be found in our database. ID: {id}"
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))

    elif fid == FORM_DELETE:

        try:
            letter = Letter.objects.filter(pk=request.POST.get("id", 0)).update(
                visibility=0
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
        except Letter.DoesNotExist:
            id = request.POST.get("id")
            user = request.user.userprofile.nick
            logger.error(
                f"There has been an attempt to delete a message with non existing id: {id}, User: {user}"
            )
            return HttpResponseRedirect(reverse("ddcz:postal-service"))
<|endoftext|>


<commit_before>"""Processes for testing purposes"""

from pywps.Process import WPSProcess
class NoInputsProcess(WPSProcess):
    """This process has no inputs and no outputs"""
    def __init__(self):
        WPSProcess.__init__(self, identifier = "noinputsprocess",title="No inputs")

class LiteralProcess(WPSProcess):
    """This process defines several types of literal type of in- and
    outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "literalprocess",
                title="Literal process",
                metadata=[{"title":"Foobar","href":"http://foo/bar"},
                          {"title":"Barfoo","href":"http://bar/foo"},
                          {"title":"Literal process"},
                          {"href":"http://foobar/"}])

        self.intIn = self.addLiteralInput(identifier="int",
                                                 title="Integer data in")

        self.stringIn = self.addLiteralInput(identifier="string",
                                                 title="String data in",
                                                 type = type(""))

        self.floatIn = self.addLiteralInput(identifier="float",
                                                 title="Float data in",
                                                 type = type(0.0))

        self.zeroInDefault = self.addLiteralInput(identifier="zerodefault",
                                                 title="Zero data input",
                                                 default=0.0,
                                                 type = type(0.0))

        self.zeroInSet = self.addLiteralInput(identifier="zeroset",
                                                 title="Zero data input",
                                                 type = type(0.0))

        self.boolIn = self.addLiteralInput(identifier="bool",
                                                 title="Boolean input",
                                                 type = type(False),
                                                 allowedValues = [True, False])

        self.intOut = self.addLiteralOutput(identifier="int",
                                                 title="Integer data out")
        self.stringOut = self.addLiteralOutput(identifier="string",
                                                 title="String data out",
                                                 type = type(""))
        self.floatOut = self.addLiteralOutput(identifier="float",
                                                 title="Float data out",
                                                 type = type(0.0))
        self.boolOut = self.addLiteralOutput(identifier="bool",
                                                 title="Boolean data out",
                                                 type = type(False))

    def execute(self):
        self.intOut.setValue(self.intIn.getValue())
        self.stringOut.setValue(self.stringIn.getValue())
        self.floatOut.setValue(self.floatIn.getValue())
        self.boolOut.setValue(self.boolIn.getValue())

class ComplexProcess(WPSProcess):
    """This process defines raster and vector data in- and outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "complexprocess",
            title="Complex process",
            storeSupported=True)

        self.vectorin = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats = [{"mimeType":"application/xml"}])

        self.rasterin = self.addComplexInput(identifier="rasterin",
                                                 title="Raster file",
                                                 formats = [{'mimeType': 'image/tiff'}, {'mimeType': 'image/geotiff'}, {'mimeType': 'application/geotiff'}, {'mimeType': 'application/x-geotiff'}, {'mimeType': 'image/png'}, {'mimeType': 'image/gif'}, {'mimeType': 'image/jpeg'}, {'mimeType': 'application/x-erdas-hfa'}, {'mimeType': 'application/netcdf'}, {'mimeType': 'application/x-netcdf'}])

        self.pausein = self.addLiteralInput(identifier="pause",
                                                 title="Pause the process",
                                                 abstract="Pause the process for several seconds, so that status=true can be tested",
                                                 default = False,
                                                 type = type(True))

        self.vectorout = self.addComplexOutput(identifier="vectorout",
                                                 title="Vector file",
                                                 formats = [{"mimeType":"text/xml"}])
        self.rasterout = self.addComplexOutput(identifier="rasterout",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}])
    def execute(self):
        self.vectorout.setValue(self.vectorin.getValue())
        self.rasterout.setValue(self.rasterin.getValue())

        if self.pausein.getValue():
            import time
            for i in range(5):
                self.status.set("Processing process",i*20)
                time.sleep(5)
        return

class ComplexProcessOWS(WPSProcess):
    def __init__(self):

        WPSProcess.__init__(self, identifier = "complexprocessows",
            title="Complex process",
            storeSupported=True)

        self.vectorin = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"}])

        self.rasterin = self.addComplexInput(identifier="rasterin",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}])

        self.pausein = self.addLiteralInput(identifier="pause",
                                                 title="Pause the process",
                                                 abstract="Pause the process for several seconds, so that status=true can be tested",
                                                 default = False,
                                                 type = type(True))

        self.vectorout = self.addComplexOutput(identifier="vectorout",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"}],
                                                 useMapscript=True)

        self.rasterout = self.addComplexOutput(identifier="rasterout",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}],
                                                 useMapscript=True)

    def execute(self):
        self.vectorout.setValue(self.vectorin.getValue())
        self.rasterout.setValue(self.rasterin.getValue())

        if self.pausein.getValue():
            import time
            for i in range(5):
                self.status.set("Processing process",i*20)
                time.sleep(5)
        return

class BBoxProcess(WPSProcess):
    """This process defines bounding box in- and outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "bboxprocess",title="BBox process")

        self.bboxin = self.addBBoxInput(identifier="bboxin",title="BBox in")
        self.bboxout = self.addBBoxOutput(identifier="bboxout",title="BBox out")

    def execute(self):
        self.bboxout.setValue(self.bboxin.value.coords)

class AssyncProcess(WPSProcess):
    """This process runs in assynchronous way"""

    def __init__(self):
        WPSProcess.__init__(self, identifier =
                "assyncprocess",title="Assynchronous process",
                storeSupported=True, statusSupported=True)
    def execute(self):
        import time
        time.sleep(2)

class FlagsProcess(WPSProcess):
    """Dummy process with --flags to test WSDL special char removal"""
    def __init__(self):
        WPSProcess.__init__(self, identifier ="flagsprocess",title="Dummy process with flags as InputOutput",storeSupported=True, statusSupported=True)
        self.flag1In = self.addLiteralInput(identifier="-flag1In",title="Literal input flag1")
        self.flag2In = self.addLiteralInput(identifier="--flag2In",title="Literal input flag2")
        self.flag1Out = self.addLiteralOutput(identifier="-flag1Out",title="Literal output flag1")
        self.flag2Out = self.addLiteralOutput(identifier="--flag2Out",title="Literal output flag2")

    def execute(self):
        self.flag1Out.setValue(self.flag1In.getValue())
        self.flag2Out.setValue(self.flag2In.getValue())

class LineageReturn(WPSProcess):
    """Lineage returning process, testing lineage with multiple inputs per identifier """
    def __init__(self):
         WPSProcess.__init__(self, identifier="lineagereturn",title="Dummy process with flags as InputOutput",storeSupported=False, statusSupported=False)
         self.vectorIn = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"},{"mimeType":"text/xml"}],
                                                 minOccurs=1,
                                                 maxOccurs=1)
         self.rasterIn = self.addComplexInput(identifier="rasterin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"image/png"},{"mimeType":"image/bmp"}],
                                                 minOccurs=1,
                                                 maxOccurs=1)
         self.bboxin = self.addBBoxInput(identifier="bboxin",title="BBox in")
         #self.vectorIn=self.addComplexInput(self,identifier="vectorin",title="vector input data",formats =[{"mimeType":"application/xml"},{"mimeType":"text/xml"}])
         #self.rasterIn=self.addComplexInput(self,identifier="rasterin",title="raster input data",formats =[{"mimeType":"image/png"},{"mimeType":"image/bmp"}])
         #self.literalOut=self.addLiteralOutput(self,identifier="literalout",title="dummy output")
    def execute(self):
        pass
        #self.literalOut.setValue("dummy")<commit_msg>Trash cleaning in lineagereturn process<commit_after>"""Processes for testing purposes"""

from pywps.Process import WPSProcess
class NoInputsProcess(WPSProcess):
    """This process has no inputs and no outputs"""
    def __init__(self):
        WPSProcess.__init__(self, identifier = "noinputsprocess",title="No inputs")

class LiteralProcess(WPSProcess):
    """This process defines several types of literal type of in- and
    outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "literalprocess",
                title="Literal process",
                metadata=[{"title":"Foobar","href":"http://foo/bar"},
                          {"title":"Barfoo","href":"http://bar/foo"},
                          {"title":"Literal process"},
                          {"href":"http://foobar/"}])

        self.intIn = self.addLiteralInput(identifier="int",
                                                 title="Integer data in")

        self.stringIn = self.addLiteralInput(identifier="string",
                                                 title="String data in",
                                                 type = type(""))

        self.floatIn = self.addLiteralInput(identifier="float",
                                                 title="Float data in",
                                                 type = type(0.0))

        self.zeroInDefault = self.addLiteralInput(identifier="zerodefault",
                                                 title="Zero data input",
                                                 default=0.0,
                                                 type = type(0.0))

        self.zeroInSet = self.addLiteralInput(identifier="zeroset",
                                                 title="Zero data input",
                                                 type = type(0.0))

        self.boolIn = self.addLiteralInput(identifier="bool",
                                                 title="Boolean input",
                                                 type = type(False),
                                                 allowedValues = [True, False])

        self.intOut = self.addLiteralOutput(identifier="int",
                                                 title="Integer data out")
        self.stringOut = self.addLiteralOutput(identifier="string",
                                                 title="String data out",
                                                 type = type(""))
        self.floatOut = self.addLiteralOutput(identifier="float",
                                                 title="Float data out",
                                                 type = type(0.0))
        self.boolOut = self.addLiteralOutput(identifier="bool",
                                                 title="Boolean data out",
                                                 type = type(False))

    def execute(self):
        self.intOut.setValue(self.intIn.getValue())
        self.stringOut.setValue(self.stringIn.getValue())
        self.floatOut.setValue(self.floatIn.getValue())
        self.boolOut.setValue(self.boolIn.getValue())

class ComplexProcess(WPSProcess):
    """This process defines raster and vector data in- and outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "complexprocess",
            title="Complex process",
            storeSupported=True)

        self.vectorin = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats = [{"mimeType":"application/xml"}])

        self.rasterin = self.addComplexInput(identifier="rasterin",
                                                 title="Raster file",
                                                 formats = [{'mimeType': 'image/tiff'}, {'mimeType': 'image/geotiff'}, {'mimeType': 'application/geotiff'}, {'mimeType': 'application/x-geotiff'}, {'mimeType': 'image/png'}, {'mimeType': 'image/gif'}, {'mimeType': 'image/jpeg'}, {'mimeType': 'application/x-erdas-hfa'}, {'mimeType': 'application/netcdf'}, {'mimeType': 'application/x-netcdf'}])

        self.pausein = self.addLiteralInput(identifier="pause",
                                                 title="Pause the process",
                                                 abstract="Pause the process for several seconds, so that status=true can be tested",
                                                 default = False,
                                                 type = type(True))

        self.vectorout = self.addComplexOutput(identifier="vectorout",
                                                 title="Vector file",
                                                 formats = [{"mimeType":"text/xml"}])
        self.rasterout = self.addComplexOutput(identifier="rasterout",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}])
    def execute(self):
        self.vectorout.setValue(self.vectorin.getValue())
        self.rasterout.setValue(self.rasterin.getValue())

        if self.pausein.getValue():
            import time
            for i in range(5):
                self.status.set("Processing process",i*20)
                time.sleep(5)
        return

class ComplexProcessOWS(WPSProcess):
    def __init__(self):

        WPSProcess.__init__(self, identifier = "complexprocessows",
            title="Complex process",
            storeSupported=True)

        self.vectorin = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"}])

        self.rasterin = self.addComplexInput(identifier="rasterin",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}])

        self.pausein = self.addLiteralInput(identifier="pause",
                                                 title="Pause the process",
                                                 abstract="Pause the process for several seconds, so that status=true can be tested",
                                                 default = False,
                                                 type = type(True))

        self.vectorout = self.addComplexOutput(identifier="vectorout",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"}],
                                                 useMapscript=True)

        self.rasterout = self.addComplexOutput(identifier="rasterout",
                                                 title="Raster file",
                                                 formats = [{"mimeType":"image/tiff"}],
                                                 useMapscript=True)

    def execute(self):
        self.vectorout.setValue(self.vectorin.getValue())
        self.rasterout.setValue(self.rasterin.getValue())

        if self.pausein.getValue():
            import time
            for i in range(5):
                self.status.set("Processing process",i*20)
                time.sleep(5)
        return

class BBoxProcess(WPSProcess):
    """This process defines bounding box in- and outputs"""

    def __init__(self):
        WPSProcess.__init__(self, identifier = "bboxprocess",title="BBox process")

        self.bboxin = self.addBBoxInput(identifier="bboxin",title="BBox in")
        self.bboxout = self.addBBoxOutput(identifier="bboxout",title="BBox out")

    def execute(self):
        self.bboxout.setValue(self.bboxin.value.coords)

class AssyncProcess(WPSProcess):
    """This process runs in assynchronous way"""

    def __init__(self):
        WPSProcess.__init__(self, identifier =
                "assyncprocess",title="Assynchronous process",
                storeSupported=True, statusSupported=True)
    def execute(self):
        import time
        time.sleep(2)

class FlagsProcess(WPSProcess):
    """Dummy process with --flags to test WSDL special char removal"""
    def __init__(self):
        WPSProcess.__init__(self, identifier ="flagsprocess",title="Dummy process with flags as InputOutput",storeSupported=True, statusSupported=True)
        self.flag1In = self.addLiteralInput(identifier="-flag1In",title="Literal input flag1")
        self.flag2In = self.addLiteralInput(identifier="--flag2In",title="Literal input flag2")
        self.flag1Out = self.addLiteralOutput(identifier="-flag1Out",title="Literal output flag1")
        self.flag2Out = self.addLiteralOutput(identifier="--flag2Out",title="Literal output flag2")

    def execute(self):
        self.flag1Out.setValue(self.flag1In.getValue())
        self.flag2Out.setValue(self.flag2In.getValue())

class LineageReturn(WPSProcess):
    """Lineage returning process, testing lineage with multiple inputs per identifier """
    def __init__(self):
         WPSProcess.__init__(self, identifier="lineagereturn",title="Dummy process with flags as InputOutput",storeSupported=False, statusSupported=False)
         self.vectorIn = self.addComplexInput(identifier="vectorin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"application/xml"},{"mimeType":"text/xml"}],
                                                 minOccurs=1,
                                                 maxOccurs=1)
         self.rasterIn = self.addComplexInput(identifier="rasterin",
                                                 title="Vector file",
                                                 formats =[{"mimeType":"image/png"},{"mimeType":"image/bmp"}],
                                                 minOccurs=1,
                                                 maxOccurs=1)
         self.bboxin = self.addBBoxInput(identifier="bboxin",title="BBox in")
    def execute(self):
        pass<|endoftext|>


<commit_before>from typing import Iterable
from re import compile
from collections import namedtuple
from subprocess import getoutput

import click

PREFIX = 'test'
SUFFIX = "g2sd"
NAME_RE = compile("menuentry '(?P<name>[\w\d\W\D]*)' -")
KERNEL_RE = compile("linux(?:[\t ]*)(?P<kernel>.*) root=(?P<root>[\w\d\-=\/]*) (?P<options>.*)")
INIT_RE = compile("initrd(?:[\t ])(?P<initrd>.*)")

MenuEntry = namedtuple('MenuEntry', 'name kernel root options initrd')

def gen_menu_entries(input: Iterable[str]) -> Iterable[MenuEntry]:
    args = []

    for num, line in enumerate(input):
        try:
            line = line.strip()

            if line.startswith('menuentry'):
                if args:
                    args = []

                name = NAME_RE.findall(line)

                args.extend(name)

            elif line.startswith("linux"):
                all = KERNEL_RE.findall(line)
                args.extend(all[0])

            elif line.startswith("initrd"):
                init = INIT_RE.findall(line)
                args.extend(init)

                yield MenuEntry(*args)

                args = []

        except Exception as ex:
            print("Error parsing file:", ex)
            exit(0)

def convert_root_entry(root_str: str) -> str:
    if root_str.startswith('UUID'):
        _, uuid = root_str.split('=')
        cmd = "blkid -t %s" % root_str

    elif root_str.startswith("PARTUUID"):
        return root_str

    else:
        cmd = "blkid %s" % root_str

    blkid_out = getoutput(cmd)
    partuuid_str = blkid_out.split(' ')[-1]
    partuuid_str = partuuid_str.replace('"', '')

    return partuuid_str

def parse_options(options_str: str) -> str:
    options = options_str.split(' ')

    return ' '.join(option for option in options
                    if not option.startswith('$'))

def menuentry_to_systemd(me: MenuEntry) -> str:
    partuuid = convert_root_entry(me.root)
    options = ' '.join((partuuid, parse_options(me.options)))

    return "title %s\nlinux %s\ninitrd %s\noptions %s" % (me.name, me.kernel, me.initrd, options)

@click.command()
@click.argument("grub_file")
@click.argument("path")
def cmd(grub_file: str, path: str):
    with open(grub_file, "r") as file:
        grub = file.read().splitlines()

    for index, me in enumerate(gen_menu_entries(grub)):
        with open(path + f'/loader/entries/{index}_{SUFFIX}.conf', 'w+') as file:
            file.write(menuentry_to_systemd(me))
            print("Wrote %s to %s" % (me.name, file.name))

if __name__ == "__main__":
    cmd()

<commit_msg>Use modern Python features<commit_after>from typing import Iterable, NamedTuple
from re import compile
from collections import namedtuple
from subprocess import getoutput
from pathlib import Path
from sys import exit

import click

PREFIX = 'test'
SUFFIX = "g2sd"
RC_PARSE_ERR = 1

NAME_RE = compile("menuentry '(?P<name>[\w\d\W\D]*)' -")
KERNEL_RE = compile("linux(?:[\t ]*)(?P<kernel>.*) root=(?P<root>[\w\d\-=\/]*) (?P<options>.*)")
INIT_RE = compile("initrd(?:[\t ])(?P<initrd>.*)")

class MenuEntry(NamedTuple):
    name: str
    kernel: str
    root: str
    options: str
    initrd: str

def gen_menu_entries(input: Iterable[str]) -> Iterable[MenuEntry]:
    args = []

    for num, line in enumerate(input):
        try:
            line = line.strip()

            if line.startswith('menuentry'):
                if args:
                    args = []

                name = NAME_RE.findall(line)

                args.extend(name)

            elif line.startswith("linux"):
                all = KERNEL_RE.findall(line)
                args.extend(all[0])

            elif line.startswith("initrd"):
                init = INIT_RE.findall(line)
                args.extend(init)

                yield MenuEntry(*args)

                args = []

        except Exception as e:
            print("Error parsing file:", e)
            exit(RC_PARSE_ERR)

def convert_root_entry(root_str: str) -> str:
    if root_str.startswith('UUID'):
        _, uuid = root_str.split('=')
        cmd = f"blkid -t {root_str}"

    elif root_str.startswith("PARTUUID"):
        return root_str

    else:
        cmd = f"blkid {root_str}"

    blkid_out = getoutput(cmd)
    partuuid_str = blkid_out.split(' ')[-1]
    partuuid_str = partuuid_str.replace('"', '')

    return partuuid_str

def parse_options(options_str: str) -> str:
    options = options_str.split(' ')

    return ' '.join(option for option in options
                    if not option.startswith('$'))

def menuentry_to_systemd(me: MenuEntry) -> str:
    partuuid = convert_root_entry(me.root)
    options = ' '.join((partuuid, parse_options(me.options)))

    return f"title {me.name}\nlinux {me.kernel}\ninitrd {me.initrd}\noptions {options}"

@click.command()
@click.argument("grub_file")
@click.argument("path")
def cmd(grub_file: str, path: str):
    grub_file = Path(grub_file)
    grub_txt = grub_file.read_text().splitlines()

    for index, me in enumerate(gen_menu_entries(grub_txt)):
        file = Path(f'/loader/entries/{index}_{SUFFIX}.conf')
        file.write_text(menuentry_to_systemd(me))
        print(f"Wrote {me.name} to {file.name}")

if __name__ == "__main__":
    cmd()

<|endoftext|>


<commit_before>import time
from operator import itemgetter

combined_g2 = {}
candidateSize = 100
termFreqs = {}
termSaliency = {}
orderedTermList = []
termRank = {}
DEFAULT_NUM_SERIATED_TERMS = 100

def reshape(term_info):
    termDistinct = {}
    termVisibility = {}
    global termFreqs, termSaliency, orderedTermList, termRank
    for element in term_info:
        term = element['term']
        orderedTermList.append( term )
        termSaliency[term] = element['saliency']
        termFreqs[term] = element['frequency']
        termDistinct[term] = element['distinctiveness']
        termRank[term] = element['rank']
        termVisibility[term] = element['visibility']

def compute( numSeriatedTerms=DEFAULT_NUM_SERIATED_TERMS):
    # Elicit from user (1) the number of terms to output and (2) a list of terms that should be included in the output...
    # set in init (i.e. read from config file)

    # Seriate!
    start_time = time.time()
    candidateTerms = orderedTermList
    term_ordering = []
    term_iter_index = []
    buffers = [0,0]

    preBest = []
    postBest = []

    for iteration in range(numSeriatedTerms):
        print("Iteration no. ", iteration)

        addedTerm = 0
        if len(term_iter_index) > 0:
            addedTerm = term_iter_index[-1]
        if iteration == 1:
            (preBest, postBest) = initBestEnergies(addedTerm, candidateTerms)
        (preBest, postBest, bestEnergies) = getBestEnergies(preBest, postBest, addedTerm)
        (candidateTerms, term_ordering, term_iter_index, buffers) = iterate_eff(candidateTerms, term_ordering, term_iter_index, buffers, bestEnergies, iteration)

        print("---------------")
        if iteration > 10:
            break
    seriation_time = time.time() - start_time

    # Output consists of (1) a list of ordered terms, and (2) the iteration index in which a term was ordered
    #print "term_ordering: ", term_ordering
    #print "term_iter_index: ", term_iter_index   # Feel free to pick a less confusing variable name

    #print "similarity matrix generation time: ", compute_sim_time
    #print "seriation time: ", seriation_time
    print("seriation time: " +  str(seriation_time))
    return term_ordering, term_iter_index

def initBestEnergies(firstTerm, candidateTerms):
    preBest = []
    postBest = []
    for candidate in candidateTerms:
        pre_score = 0
        post_score = 0

        # preBest
        if (candidate, firstTerm) in combined_g2:
            pre_score = combined_g2[(candidate, firstTerm)]
        # postBest
        if (firstTerm, candidate) in combined_g2:
            post_score = combined_g2[(firstTerm, candidate)]

        preBest.append((candidate, pre_score))
        postBest.append((candidate, post_score))

    return (preBest, postBest)

def getBestEnergies(preBest, postBest, addedTerm):
    if addedTerm == 0:
        return (preBest, postBest, [])

    term_order = [x[0] for x in preBest]
    # compare candidate terms' bests against newly added term
    remove_index = -1
    for existingIndex in range(len(preBest)):
        term = term_order[existingIndex]
        if term == addedTerm:
            remove_index = existingIndex

        # check pre energies
        if (term, addedTerm) in combined_g2:
            if combined_g2[(term, addedTerm)] > preBest[existingIndex][1]:
                preBest[existingIndex] = (term, combined_g2[(term, addedTerm)])
        # check post energies
        if (addedTerm, term) in combined_g2:
            if combined_g2[(addedTerm, term)] > postBest[existingIndex][1]:
                postBest[existingIndex] = (term, combined_g2[(addedTerm, term)])

    # remove the added term's preBest and postBest scores
    if remove_index != -1:
        del preBest[remove_index]
        del postBest[remove_index]

    #create and sort the bestEnergies list
    energyMax = [sum(pair) for pair in zip([x[1] for x in preBest], [y[1] for y in postBest])]
    bestEnergies = zip([x[0] for x in preBest], energyMax)

    return (preBest, postBest, sorted(bestEnergies, key=itemgetter(1), reverse=True))

def iterate_eff( candidateTerms, term_ordering, term_iter_index, buffers, bestEnergies, iteration_no ):
    maxEnergyChange = 0.0;
    maxTerm = "";
    maxPosition = 0;

    if len(bestEnergies) != 0:
        bestEnergy_terms = [x[0] for x in bestEnergies]
    else:
        bestEnergy_terms = candidateTerms

    breakout_counter = 0
    for candidate_index in range(len(bestEnergy_terms)):
        breakout_counter += 1
        candidate = bestEnergy_terms[candidate_index]
        for position in range(len(term_ordering)+1):
            current_buffer = buffers[position]
            candidateRank = termRank[candidate]
            if candidateRank <= (len(term_ordering) + candidateSize):
                current_energy_change = getEnergyChange(candidate, position, term_ordering, current_buffer, iteration_no)
                if current_energy_change > maxEnergyChange:
                    maxEnergyChange = current_energy_change
                    maxTerm = candidate
                    maxPosition = position
                    # check for early termination
        if candidate_index < len(bestEnergy_terms)-1 and len(bestEnergies) != 0:
            if maxEnergyChange >= (2*(bestEnergies[candidate_index][1] + current_buffer)):
                print("#-------- breaking out early ---------#")
                print("candidates checked: ", breakout_counter)
                break;

    print("change in energy: ", maxEnergyChange)
    print("maxTerm: ", maxTerm)
    print("maxPosition: ", maxPosition)

    candidateTerms.remove(maxTerm)

    # update buffers
    buf_score = 0
    if len(term_ordering) == 0:
        # what a waaat?
        buffers = buffers
    elif maxPosition >= len(term_ordering):
        if (term_ordering[-1], maxTerm) in combined_g2:
            buf_score = combined_g2[(term_ordering[-1], maxTerm)]
        buffers.insert(len(buffers)-1, buf_score)
    elif maxPosition == 0:
        if (maxTerm, term_ordering[0]) in combined_g2:
            buf_score = combined_g2[(maxTerm, term_ordering[0])]
        buffers.insert(1, buf_score)
    else:
        if (term_ordering[maxPosition-1], maxTerm) in combined_g2:
            buf_score = combined_g2[(term_ordering[maxPosition-1], maxTerm)]
        buffers[maxPosition] = buf_score

        buf_score = 0
        if (maxTerm, term_ordering[maxPosition]) in combined_g2:
            buf_score = combined_g2[(maxTerm, term_ordering[maxPosition])]
        buffers.insert(maxPosition+1, buf_score)

    # update term ordering and ranking
    if maxPosition >= len(term_ordering):
        term_ordering.append(maxTerm)
    else:
        term_ordering.insert(maxPosition, maxTerm)
    term_iter_index.append(maxTerm)
    return (candidateTerms, term_ordering, term_iter_index, buffers)

def getEnergyChange(candidateTerm, position, term_list, currentBuffer, iteration_no):
    prevBond = 0.0
    postBond = 0.0

    # first iteration only
    if iteration_no == 0:
        current_freq = 0.0
        current_saliency = 0.0

        if candidateTerm in termFreqs:
            current_freq = termFreqs[candidateTerm]
        if candidateTerm in termSaliency:
            current_saliency = termSaliency[candidateTerm]
        return 0.001 * current_freq * current_saliency

    # get previous term
    if position > 0:
        prev_term = term_list[position-1]
        if (prev_term, candidateTerm) in combined_g2:
            prevBond = combined_g2[(prev_term, candidateTerm)]

    # get next term
    if position < len(term_list):
        next_term = term_list[position]
        if (next_term, candidateTerm) in combined_g2:
            postBond = combined_g2[(candidateTerm, next_term)]

    return 2*(prevBond + postBond - currentBuffer)
<commit_msg>Fixes to make seriation process compatible with script.<commit_after>from operator import itemgetter
candidateSize = 100
def initBestEnergies(combined_g2, firstTerm, candidateTerms):
    preBest = []
    postBest = []
    for candidate in candidateTerms:
        pre_score = 0
        post_score = 0

        # preBest
        if (candidate, firstTerm) in combined_g2:
            pre_score = combined_g2[(candidate, firstTerm)]
        # postBest
        if (firstTerm, candidate) in combined_g2:
            post_score = combined_g2[(firstTerm, candidate)]

        preBest.append((candidate, pre_score))
        postBest.append((candidate, post_score))

    return (preBest, postBest)

def getBestEnergies(combined_g2, preBest, postBest, addedTerm):
    if addedTerm == 0:
        return (preBest, postBest, [])

    term_order = [x[0] for x in preBest]
    # compare candidate terms' bests against newly added term
    remove_index = -1
    for existingIndex in range(len(preBest)):
        term = term_order[existingIndex]
        if term == addedTerm:
            remove_index = existingIndex

        # check pre energies
        if (term, addedTerm) in combined_g2:
            if combined_g2[(term, addedTerm)] > preBest[existingIndex][1]:
                preBest[existingIndex] = (term, combined_g2[(term, addedTerm)])
        # check post energies
        if (addedTerm, term) in combined_g2:
            if combined_g2[(addedTerm, term)] > postBest[existingIndex][1]:
                postBest[existingIndex] = (term, combined_g2[(addedTerm, term)])

    # remove the added term's preBest and postBest scores
    if remove_index != -1:
        del preBest[remove_index]
        del postBest[remove_index]

    #create and sort the bestEnergies list
    energyMax = [sum(pair) for pair in zip([x[1] for x in preBest], [y[1] for y in postBest])]
    bestEnergies = zip([x[0] for x in preBest], energyMax)

    return (preBest, postBest, sorted(bestEnergies, key=itemgetter(1), reverse=True))

def iterate_eff( combined_g2, termRank, termFreqs, termSaliency, candidateTerms, term_ordering, term_iter_index, buffers, bestEnergies, iteration_no ):
    maxEnergyChange = 0.0;
    maxTerm = "";
    maxPosition = 0;

    if len(bestEnergies) != 0:
        bestEnergy_terms = [x[0] for x in bestEnergies]
    else:
        bestEnergy_terms = candidateTerms

    breakout_counter = 0
    for candidate_index in range(len(bestEnergy_terms)):
        breakout_counter += 1
        candidate = bestEnergy_terms[candidate_index]
        for position in range(len(term_ordering)+1):
            current_buffer = buffers[position]
            candidateRank = termRank[candidate]
            if candidateRank <= (len(term_ordering) + candidateSize):
                current_energy_change = getEnergyChange(combined_g2, termFreqs, termSaliency, candidate, position, term_ordering, current_buffer, iteration_no)
                if current_energy_change > maxEnergyChange:
                    maxEnergyChange = current_energy_change
                    maxTerm = candidate
                    maxPosition = position
                    # check for early termination
        if candidate_index < len(bestEnergy_terms)-1 and len(bestEnergies) != 0:
            if maxEnergyChange >= (2*(bestEnergies[candidate_index][1] + current_buffer)):
                print("#-------- breaking out early ---------#")
                print("candidates checked: ", breakout_counter)
                break;

    print("change in energy: ", maxEnergyChange)
    print("maxTerm: ", maxTerm)
    print("maxPosition: ", maxPosition)

    candidateTerms.remove(maxTerm)

    # update buffers
    buf_score = 0
    if len(term_ordering) == 0:
        # what a waaat?
        buffers = buffers
    elif maxPosition >= len(term_ordering):
        if (term_ordering[-1], maxTerm) in combined_g2:
            buf_score = combined_g2[(term_ordering[-1], maxTerm)]
        buffers.insert(len(buffers)-1, buf_score)
    elif maxPosition == 0:
        if (maxTerm, term_ordering[0]) in combined_g2:
            buf_score = combined_g2[(maxTerm, term_ordering[0])]
        buffers.insert(1, buf_score)
    else:
        if (term_ordering[maxPosition-1], maxTerm) in combined_g2:
            buf_score = combined_g2[(term_ordering[maxPosition-1], maxTerm)]
        buffers[maxPosition] = buf_score

        buf_score = 0
        if (maxTerm, term_ordering[maxPosition]) in combined_g2:
            buf_score = combined_g2[(maxTerm, term_ordering[maxPosition])]
        buffers.insert(maxPosition+1, buf_score)

    # update term ordering and ranking
    if maxPosition >= len(term_ordering):
        term_ordering.append(maxTerm)
    else:
        term_ordering.insert(maxPosition, maxTerm)
    term_iter_index.append(maxTerm)
    return (candidateTerms, term_ordering, term_iter_index, buffers)

def getEnergyChange(combined_g2, termFreqs, termSaliency, candidateTerm, position, term_list, currentBuffer, iteration_no):
    prevBond = 0.0
    postBond = 0.0

    # first iteration only
    if iteration_no == 0:
        current_freq = 0.0
        current_saliency = 0.0

        if candidateTerm in termFreqs:
            current_freq = termFreqs[candidateTerm]
        if candidateTerm in termSaliency:
            current_saliency = termSaliency[candidateTerm]
        return 0.001 * current_freq * current_saliency

    # get previous term
    if position > 0:
        prev_term = term_list[position-1]
        if (prev_term, candidateTerm) in combined_g2:
            prevBond = combined_g2[(prev_term, candidateTerm)]

    # get next term
    if position < len(term_list):
        next_term = term_list[position]
        if (next_term, candidateTerm) in combined_g2:
            postBond = combined_g2[(candidateTerm, next_term)]

    return 2*(prevBond + postBond - currentBuffer)
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import logging

import sys

import os
from adapters import ipprovider_opendns, UpdateDnsFailed, IpProviderFailure
from adapters import ddnsprovider_aws
from adapters.config import Config

logger = logging.getLogger(__name__)

class DdnsUpdater(object):
    def __init__(self, ip_provider=None, ddns_provider=None, config_provider=None):
        self._ip_address_provider = ip_provider or ipprovider_opendns.get_ip_address
        self._ddns_provider = ddns_provider or ddnsprovider_aws.update_dns
        self._config_provider = config_provider
        if not config_provider:
            try:
                self._config_provider = Config(os.path.join(os.path.dirname(__file__), "ddns_updater_aws.ini"))
            except IOError as e:
                logger.exception("Unable to load config file")

    def run(self):
        logger.info("Discovering ip address...")
        ipprovider_config = self._config_provider.get_opendns_config()
        try:
            ip = self._ip_address_provider(ipprovider_config)
        except IpProviderFailure:
            logger.error("no IP address was discovered")
            return

        logger.info("ip [{}] was discovered, sending to ddns provider ...".format(ip))
        aws_config = self._config_provider.get_aws_config()
        try:
            self._ddns_provider(ip, aws_config)
            logger.info('ddns provider updated !')
        except UpdateDnsFailed as e:
            logger.exception(e)
            logger.error("Could not update dns: {}".format(e.message))

def main():  # pragma: no cover
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)
    logging.getLogger().setLevel(logging.INFO)
    ddns_updater = DdnsUpdater()
    ddns_updater.run()

if __name__ == "__main__":
    main()
<commit_msg>ignore main() for coverage<commit_after>#!/usr/bin/env python2

import logging

import sys

import os
from adapters import ipprovider_opendns, UpdateDnsFailed, IpProviderFailure
from adapters import ddnsprovider_aws
from adapters.config import Config

logger = logging.getLogger(__name__)

class DdnsUpdater(object):
    def __init__(self, ip_provider=None, ddns_provider=None, config_provider=None):
        self._ip_address_provider = ip_provider or ipprovider_opendns.get_ip_address
        self._ddns_provider = ddns_provider or ddnsprovider_aws.update_dns
        self._config_provider = config_provider
        if not config_provider:
            try:
                self._config_provider = Config(os.path.join(os.path.dirname(__file__), "ddns_updater_aws.ini"))
            except IOError as e:
                logger.exception("Unable to load config file")

    def run(self):
        logger.info("Discovering ip address...")
        ipprovider_config = self._config_provider.get_opendns_config()
        try:
            ip = self._ip_address_provider(ipprovider_config)
        except IpProviderFailure:
            logger.error("no IP address was discovered")
            return

        logger.info("ip [{}] was discovered, sending to ddns provider ...".format(ip))
        aws_config = self._config_provider.get_aws_config()
        try:
            self._ddns_provider(ip, aws_config)
            logger.info('ddns provider updated !')
        except UpdateDnsFailed as e:
            logger.exception(e)
            logger.error("Could not update dns: {}".format(e.message))

def main():  # pragma: no cover
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)
    logging.getLogger().setLevel(logging.INFO)
    ddns_updater = DdnsUpdater()
    ddns_updater.run()

if __name__ == "__main__":   # pragma: no cover
    main()
<|endoftext|>