<commit_before># Copyright 2013 Daniel Narvaez
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
from subprocess import check_call

from broot.builder import FedoraBuilder
from broot.builder import DebianBuilder

class Root:
    def __init__(self, config):
        self.path = os.path.abspath(config["path"])

        self._config = config
        self._mounts = self._compute_mounts()
        self._user_name = "broot"

        distro = config.get("distro", "debian")

        if distro == "debian":
            self._builder = DebianBuilder(self)
        elif distro == "fedora":
            self._builder = FedoraBuilder(self)
        else:
            raise ValueError("Unknown distro %s" % distro)

    def _compute_mounts(self):
        mounts = {}

        for source_path, dest_path in self._config.get("mounts", {}).items():
            mounts[os.path.abspath(source_path)] = dest_path

        for source_path in ["/dev", "/dev/pts", "/dev/shm", "/sys", "/proc",
                            "/tmp"]:
            mounts[source_path] = os.path.join(self.path, source_path[1:])

        return mounts

    def activate(self):
        self._mounted = []

        for source_path, dest_path in self._mounts.items():
            check_call(["mount", "--bind", source_path, dest_path])
            self._mounted.append(dest_path)

        shutil.copyfile(os.path.join("/etc", "resolv.conf"),
                        os.path.join(self.path, "etc", "resolv.conf"))

    def deactivate(self):
        for mount_path in reversed(self._mounted):
            check_call(["umount", mount_path])

        del self._mounted

    def install_packages(self, packages):
        self._builder.install_packages(packages)

    def create(self, mirror=None):
        try:
            os.makedirs(self.path)
        except OSError:
            pass

        self._builder.create(mirror)

        self._setup_bashrc("root")

        self._create_user()
        self._setup_bashrc(os.path.join("home", self._user_name))

    def run(self, command, root=False):
        if root:
            chroot = "chroot"
        else:
            chroot = "chroot --userspec %s:%s" % (
                self._user_name, self._user_name)

        check_call("%s %s /bin/bash -lc \"%s\"" %
                   (chroot, self.path, command), shell=True)

    def _create_user(self):
        gid = os.environ["SUDO_GID"]

        self.run("/usr/sbin/addgroup %s --gid %s" % (self._user_name, gid),
                 root=True)

        self.run("/usr/sbin/adduser %s --uid %s --gid %s" %
                 (self._user_name, os.environ["SUDO_UID"], gid), root=True)

    def _setup_bashrc(self, home_path):
        environ = {"LANG": "C"}

        with open(os.path.join(self.path, home_path, ".bashrc"), "w") as f:
            for variable, value in environ.items():
                f.write("export %s=%s\n" % (variable, value))
<commit_msg>Invert source/dest path<commit_after># Copyright 2013 Daniel Narvaez
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
from subprocess import check_call

from broot.builder import FedoraBuilder
from broot.builder import DebianBuilder

class Root:
    def __init__(self, config):
        self.path = os.path.abspath(config["path"])

        self._config = config
        self._mounts = self._compute_mounts()
        self._user_name = "broot"

        distro = config.get("distro", "debian")

        if distro == "debian":
            self._builder = DebianBuilder(self)
        elif distro == "fedora":
            self._builder = FedoraBuilder(self)
        else:
            raise ValueError("Unknown distro %s" % distro)

    def _compute_mounts(self):
        mounts = {}

        for source_path, dest_path in self._config.get("mounts", {}).items():
            mounts[source_path] = os.path.abspath(dest_path)

        for source_path in ["/dev", "/dev/pts", "/dev/shm", "/sys", "/proc",
                            "/tmp"]:
            mounts[source_path] = os.path.join(self.path, source_path[1:])

        return mounts

    def activate(self):
        self._mounted = []

        for source_path, dest_path in self._mounts.items():
            check_call(["mount", "--bind", source_path, dest_path])
            self._mounted.append(dest_path)

        shutil.copyfile(os.path.join("/etc", "resolv.conf"),
                        os.path.join(self.path, "etc", "resolv.conf"))

    def deactivate(self):
        for mount_path in reversed(self._mounted):
            check_call(["umount", mount_path])

        del self._mounted

    def install_packages(self, packages):
        self._builder.install_packages(packages)

    def create(self, mirror=None):
        try:
            os.makedirs(self.path)
        except OSError:
            pass

        self._builder.create(mirror)

        self._setup_bashrc("root")

        self._create_user()
        self._setup_bashrc(os.path.join("home", self._user_name))

    def run(self, command, root=False):
        if root:
            chroot = "chroot"
        else:
            chroot = "chroot --userspec %s:%s" % (
                self._user_name, self._user_name)

        check_call("%s %s /bin/bash -lc \"%s\"" %
                   (chroot, self.path, command), shell=True)

    def _create_user(self):
        gid = os.environ["SUDO_GID"]

        self.run("/usr/sbin/addgroup %s --gid %s" % (self._user_name, gid),
                 root=True)

        self.run("/usr/sbin/adduser %s --uid %s --gid %s" %
                 (self._user_name, os.environ["SUDO_UID"], gid), root=True)

    def _setup_bashrc(self, home_path):
        environ = {"LANG": "C"}

        with open(os.path.join(self.path, home_path, ".bashrc"), "w") as f:
            for variable, value in environ.items():
                f.write("export %s=%s\n" % (variable, value))
<|endoftext|>


<commit_before>

# Third-Party
import django_rq

# Django
from django.apps import apps
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.functional import cached_property

# First-Party
from bhs.managers import HumanManager
from bhs.managers import JoinManager
from bhs.managers import RoleManager
from bhs.managers import StructureManager
# from bhs.managers import SubscriptionManager

class Human(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    first_name = models.CharField(
        max_length=255,
        editable=False,
    )
    middle_name = models.CharField(
        max_length=255,
        editable=False,
        db_column='middle_initial',
    )
    last_name = models.CharField(
        max_length=255,
        editable=False,
    )
    nick_name = models.CharField(
        max_length=255,
        editable=False,
        db_column='preferred_name',
    )
    email = models.CharField(
        max_length=255,
        unique=True,
        editable=False,
        db_column='username',
    )
    birth_date = models.DateField(
        editable=False,
        null=True,
        db_column='birthday'
    )
    is_deceased = models.BooleanField(
        editable=False,
    )
    phone = models.CharField(
        max_length=255,
        editable=False,
    )
    cell_phone = models.CharField(
        max_length=255,
        editable=False,
    )
    work_phone = models.CharField(
        max_length=255,
        editable=False,
    )
    bhs_id = models.IntegerField(
        editable=False,
        unique=True,
        null=False,
        db_column='legacy_id',
    )
    sex = models.CharField(
        max_length=255,
        editable=False,
    )
    primary_voice_part = models.CharField(
        max_length=255,
        editable=False,
    )

    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    objects = HumanManager()

    # Properties
    @cached_property
    def full_name(self):
        if self.first_name:
            first_name = self.first_name.strip()
        else:
            first_name = None
        if self.middle_name:
            middle_name = self.middle_name.strip()
        else:
            middle_name = None
        if self.last_name:
            last_name = self.last_name.strip()
        else:
            last_name = None
        if self.nick_name:
            format_nick = "({0})".format(
                self.nick_name.replace("'", "").replace('"', '').replace("(", "").replace(")", "").strip()
            )
        else:
            format_nick = None
        full_name = " ".join(
            filter(
                None, [
                    first_name,
                    middle_name,
                    last_name,
                    format_nick,
                ]
            )
        )
        return full_name

    # Internals
    def __str__(self):
        return "{0} [{1}]".format(
            self.full_name,
            self.bhs_id,
        )

    def update_person(self):
        Person = apps.get_model('api.person')
        return Person.objects.update_or_create_from_human(self)

    # def update_user(self):
    #     Subscription = apps.get_model('bhs.subscription')
    #     User = apps.get_model('api.user')
    #     queue = django_rq.get_queue('low')
    #     try:
    #         subscription = self.subscriptions.filter(
    #             items_editable=True,
    #         ).latest('modified')
    #         queue.enqueue(
    #             User.objects.update_or_create_from_subscription,
    #             subscription,
    #         )
    #     except Subscription.DoesNotExist:
    #         pass

    class Meta:
        managed=False
        db_table = 'vwMembers'

class Structure(models.Model):
    CHAPTER = 'chapter'
    DISTRICT = 'district'
    ORGANIZATION = 'organization'
    QUARTET = 'quartet'
    CHORUS = 'chorus'
    GROUP = 'group'

    KIND = [
        (ORGANIZATION, 'Organization'),
        (DISTRICT, 'District'),
        (GROUP, 'Group'),
        (CHAPTER, 'Chapter'),
        (CHORUS, 'Chorus'),
        (QUARTET, 'Quartet'),
    ]

    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    kind = models.CharField(
        max_length=255,
        editable=False,
        choices=KIND,
        db_column='object_type',
    )
    category = models.CharField(
        max_length=255,
        editable=False,
    )
    bhs_id = models.IntegerField(
        editable=False,
        unique=True,
        null=False,
        db_column='legacy_id',
    )
    chapter_code = models.CharField(
        max_length=255,
        editable=False,
        db_column='legacy_code',
    )
    chorus_name = models.CharField(
        max_length=255,
        editable=False,
    )
    preferred_name = models.CharField(
        max_length=255,
        editable=False,
    )
    phone = models.CharField(
        max_length=255,
        editable=False,
    )
    email = models.CharField(
        max_length=255,
        editable=False,
    )
    website = models.CharField(
        max_length=255,
        editable=False,
    )
    facebook = models.CharField(
        max_length=255,
        editable=False,
    )
    twitter = models.CharField(
        max_length=255,
        editable=False,
    )
    established_date = models.DateField(
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created',
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        editable=False,
    )
    # FKs
    status = models.ForeignKey(
        'Status',
        related_name='structures',
        editable=False,
        on_delete=models.CASCADE,
    )

    parent = models.ForeignKey(
        'self',
        null=True,
        blank=True,
        related_name='children',
        db_column='parent_id',
        on_delete=models.CASCADE,
    )

    objects = StructureManager()

    def __str__(self):
        if self.name:
            name = self.name.strip()
        else:
            name = 'UNKNOWN'
        return "{0} [{1}]".format(
            name,
            self.bhs_id,
        )

    def update_bs(self):
        Group = apps.get_model('api.group')
        Member = apps.get_model('api.member')
        Officer = apps.get_model('api.officer')
        Group.objects.update_or_create_from_structure(self)
        joins = self.get_joins()
        for join in joins:
            Member.objects.update_or_create_from_join(join)
        roles = self.get_roles()
        for role in roles:
            Officer.objects.update_or_create_from_role(role)
        return

    def get_joins(self):
        humans = self.joins.select_related(
            'subscription__human',
        ).values_list(
            'subscription__human',
            flat=True,
        ).distinct()
        joins = []
        for human in humans:
            join = self.joins.select_related(
                'subscription',
                'subscription__human',
            ).filter(
                subscription__human__id=human,
            ).latest(
                'modified',
                '-inactive_date',
            )
            joins.append(join)
        return joins

    def get_roles(self):
        pairs = self.roles.select_related(
            'human',
        ).values_list(
            'human',
            'name',
        ).distinct()
        roles = []
        for human, name in pairs:
            role = self.roles.select_related(
                'human',
            ).filter(
                human__id=human,
                name=name,
            ).latest(
                'modified',
                '-end_date',
            )
            roles.append(role)
        return roles

    class Meta:
        managed=False
        db_table = 'vwStructures'

class Status(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    # Internals

    def __str__(self):
        return str(self.name)

    class Meta:
        managed=False
        db_table = 'vwStatuses'
        verbose_name_plural = 'statuses'

class Membership(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )

    code = models.CharField(
        max_length=255,
        editable=False,
    )

    created = models.DateTimeField(
        db_column='created',
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='modified',
        editable=False,
    )
    # FKs
    structure = models.ForeignKey(
        'Structure',
        related_name='memberships',
        editable=False,
        db_column='object_id',
        on_delete=models.CASCADE,
    )
    status = models.ForeignKey(
        'Status',
        related_name='memberships',
        editable=False,
        on_delete=models.CASCADE,
    )

    # Internals
    def __str__(self):
        return "{0} {1}".format(
            self.structure,
            self.code,
        )

    class Meta:
        managed=False
        db_table = 'vwMemberships'

class Subscription(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    current_through = models.DateField(
        db_column='valid_through',
        null=True,
        editable=False,
    )
    status = models.CharField(
        max_length=255,
        editable=False,
    )
    items_editable = models.BooleanField(
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    # objects = SubscriptionManager()

    # FKs
    human = models.ForeignKey(
        'Human',
        related_name='subscriptions',
        editable=False,
        db_column='members_id',
        on_delete=models.CASCADE,
    )

    # Internals
    def __str__(self):
        return str(self.human)

    class Meta:
        managed = False
        db_table = 'vwSubscriptions'

class Role(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    abbv = models.CharField(
        max_length=255,
        editable=False,
    )
    officer_roles_id = models.CharField(
        max_length=255,
        editable=False,
    )
    start_date = models.DateField(
        null=True,
        editable=False,
    )
    end_date = models.DateField(
        null=True,
        editable=False,
    )
    # FKs
    human = models.ForeignKey(
        'Human',
        related_name='roles',
        editable=False,
        db_column='member_id',
        on_delete=models.CASCADE,
    )
    structure = models.ForeignKey(
        'Structure',
        related_name='roles',
        editable=False,
        db_column='object_id',
        on_delete=models.CASCADE,
    )

    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    objects = RoleManager()

    # Internals
    def __str__(self):
        return "{0} {1} {2}".format(
            self.name,
            self.human,
            self.structure,
        )

    class Meta:
        managed=False
        db_table = 'vwOfficers'

class Join(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    status = models.BooleanField(
        editable=False,
    )
    paid = models.BooleanField(
        editable=False,
    )
    vocal_part = models.CharField(
        max_length=255,
        editable=False,
    )
    established_date = models.DateField(
        db_column='created',
        null=False,
        editable=False,
    )
    inactive_date = models.DateField(
        db_column='inactive',
        null=True,
        editable=False,
    )
    inactive_reason = models.CharField(
        max_length=255,
        db_column='inactive_reason',
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created_on',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='modified',
        null=True,
        editable=False,
    )

    objects = JoinManager()

    # FKs
    subscription = models.ForeignKey(
        'Subscription',
        editable=False,
        related_name='joins',
        on_delete=models.CASCADE,
    )
    membership = models.ForeignKey(
        'Membership',
        editable=False,
        related_name='joins',
        on_delete=models.CASCADE,
    )

    structure = models.ForeignKey(
        'Structure',
        editable=False,
        related_name='joins',
        db_column='reference_structure_id',
        on_delete=models.CASCADE,
    )

    # Internals
    def clean(self):
        if all([
            not self.inactive_date,
            self.subscription.items_editable,
            self.subscription.status == 'expired',
        ]):
            raise ValidationError({
                'inactive_date': 'Inactive Date is missing on expired subscription.',
            })

    def __str__(self):
        return str(self.id)
        # return "{0} {1}".format(
        #     self.subscription,
        #     self.membership,
        # )

    class Meta:
        managed = False
        db_table = 'vwSubscriptions_Memberships'
        verbose_name = 'Join'
        verbose_name_plural = 'Joins'
<commit_msg>Only filter paid join rows<commit_after>

# Third-Party
import django_rq

# Django
from django.apps import apps
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.functional import cached_property

# First-Party
from bhs.managers import HumanManager
from bhs.managers import JoinManager
from bhs.managers import RoleManager
from bhs.managers import StructureManager
# from bhs.managers import SubscriptionManager

class Human(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    first_name = models.CharField(
        max_length=255,
        editable=False,
    )
    middle_name = models.CharField(
        max_length=255,
        editable=False,
        db_column='middle_initial',
    )
    last_name = models.CharField(
        max_length=255,
        editable=False,
    )
    nick_name = models.CharField(
        max_length=255,
        editable=False,
        db_column='preferred_name',
    )
    email = models.CharField(
        max_length=255,
        unique=True,
        editable=False,
        db_column='username',
    )
    birth_date = models.DateField(
        editable=False,
        null=True,
        db_column='birthday'
    )
    is_deceased = models.BooleanField(
        editable=False,
    )
    phone = models.CharField(
        max_length=255,
        editable=False,
    )
    cell_phone = models.CharField(
        max_length=255,
        editable=False,
    )
    work_phone = models.CharField(
        max_length=255,
        editable=False,
    )
    bhs_id = models.IntegerField(
        editable=False,
        unique=True,
        null=False,
        db_column='legacy_id',
    )
    sex = models.CharField(
        max_length=255,
        editable=False,
    )
    primary_voice_part = models.CharField(
        max_length=255,
        editable=False,
    )

    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    objects = HumanManager()

    # Properties
    @cached_property
    def full_name(self):
        if self.first_name:
            first_name = self.first_name.strip()
        else:
            first_name = None
        if self.middle_name:
            middle_name = self.middle_name.strip()
        else:
            middle_name = None
        if self.last_name:
            last_name = self.last_name.strip()
        else:
            last_name = None
        if self.nick_name:
            format_nick = "({0})".format(
                self.nick_name.replace("'", "").replace('"', '').replace("(", "").replace(")", "").strip()
            )
        else:
            format_nick = None
        full_name = " ".join(
            filter(
                None, [
                    first_name,
                    middle_name,
                    last_name,
                    format_nick,
                ]
            )
        )
        return full_name

    # Internals
    def __str__(self):
        return "{0} [{1}]".format(
            self.full_name,
            self.bhs_id,
        )

    def update_person(self):
        Person = apps.get_model('api.person')
        return Person.objects.update_or_create_from_human(self)

    # def update_user(self):
    #     Subscription = apps.get_model('bhs.subscription')
    #     User = apps.get_model('api.user')
    #     queue = django_rq.get_queue('low')
    #     try:
    #         subscription = self.subscriptions.filter(
    #             items_editable=True,
    #         ).latest('modified')
    #         queue.enqueue(
    #             User.objects.update_or_create_from_subscription,
    #             subscription,
    #         )
    #     except Subscription.DoesNotExist:
    #         pass

    class Meta:
        managed=False
        db_table = 'vwMembers'

class Structure(models.Model):
    CHAPTER = 'chapter'
    DISTRICT = 'district'
    ORGANIZATION = 'organization'
    QUARTET = 'quartet'
    CHORUS = 'chorus'
    GROUP = 'group'

    KIND = [
        (ORGANIZATION, 'Organization'),
        (DISTRICT, 'District'),
        (GROUP, 'Group'),
        (CHAPTER, 'Chapter'),
        (CHORUS, 'Chorus'),
        (QUARTET, 'Quartet'),
    ]

    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    kind = models.CharField(
        max_length=255,
        editable=False,
        choices=KIND,
        db_column='object_type',
    )
    category = models.CharField(
        max_length=255,
        editable=False,
    )
    bhs_id = models.IntegerField(
        editable=False,
        unique=True,
        null=False,
        db_column='legacy_id',
    )
    chapter_code = models.CharField(
        max_length=255,
        editable=False,
        db_column='legacy_code',
    )
    chorus_name = models.CharField(
        max_length=255,
        editable=False,
    )
    preferred_name = models.CharField(
        max_length=255,
        editable=False,
    )
    phone = models.CharField(
        max_length=255,
        editable=False,
    )
    email = models.CharField(
        max_length=255,
        editable=False,
    )
    website = models.CharField(
        max_length=255,
        editable=False,
    )
    facebook = models.CharField(
        max_length=255,
        editable=False,
    )
    twitter = models.CharField(
        max_length=255,
        editable=False,
    )
    established_date = models.DateField(
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created',
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        editable=False,
    )
    # FKs
    status = models.ForeignKey(
        'Status',
        related_name='structures',
        editable=False,
        on_delete=models.CASCADE,
    )

    parent = models.ForeignKey(
        'self',
        null=True,
        blank=True,
        related_name='children',
        db_column='parent_id',
        on_delete=models.CASCADE,
    )

    objects = StructureManager()

    def __str__(self):
        if self.name:
            name = self.name.strip()
        else:
            name = 'UNKNOWN'
        return "{0} [{1}]".format(
            name,
            self.bhs_id,
        )

    def update_bs(self):
        Group = apps.get_model('api.group')
        Member = apps.get_model('api.member')
        Officer = apps.get_model('api.officer')
        Group.objects.update_or_create_from_structure(self)
        joins = self.get_joins()
        for join in joins:
            Member.objects.update_or_create_from_join(join)
        roles = self.get_roles()
        for role in roles:
            Officer.objects.update_or_create_from_role(role)
        return

    def get_joins(self):
        humans = self.joins.select_related(
            'subscription__human',
        ).values_list(
            'subscription__human',
            flat=True,
        ).distinct()
        joins = []
        for human in humans:
            join = self.joins.select_related(
                'subscription',
                'subscription__human',
            ).filter(
                paid=True,
                subscription__human__id=human,
            ).latest(
                'modified',
                '-inactive_date',
            )
            joins.append(join)
        return joins

    def get_roles(self):
        pairs = self.roles.select_related(
            'human',
        ).values_list(
            'human',
            'name',
        ).distinct()
        roles = []
        for human, name in pairs:
            role = self.roles.select_related(
                'human',
            ).filter(
                human__id=human,
                name=name,
            ).latest(
                'modified',
                '-end_date',
            )
            roles.append(role)
        return roles

    class Meta:
        managed=False
        db_table = 'vwStructures'

class Status(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    # Internals

    def __str__(self):
        return str(self.name)

    class Meta:
        managed=False
        db_table = 'vwStatuses'
        verbose_name_plural = 'statuses'

class Membership(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )

    code = models.CharField(
        max_length=255,
        editable=False,
    )

    created = models.DateTimeField(
        db_column='created',
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='modified',
        editable=False,
    )
    # FKs
    structure = models.ForeignKey(
        'Structure',
        related_name='memberships',
        editable=False,
        db_column='object_id',
        on_delete=models.CASCADE,
    )
    status = models.ForeignKey(
        'Status',
        related_name='memberships',
        editable=False,
        on_delete=models.CASCADE,
    )

    # Internals
    def __str__(self):
        return "{0} {1}".format(
            self.structure,
            self.code,
        )

    class Meta:
        managed=False
        db_table = 'vwMemberships'

class Subscription(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    current_through = models.DateField(
        db_column='valid_through',
        null=True,
        editable=False,
    )
    status = models.CharField(
        max_length=255,
        editable=False,
    )
    items_editable = models.BooleanField(
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    # objects = SubscriptionManager()

    # FKs
    human = models.ForeignKey(
        'Human',
        related_name='subscriptions',
        editable=False,
        db_column='members_id',
        on_delete=models.CASCADE,
    )

    # Internals
    def __str__(self):
        return str(self.human)

    class Meta:
        managed = False
        db_table = 'vwSubscriptions'

class Role(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    name = models.CharField(
        max_length=255,
        editable=False,
    )
    abbv = models.CharField(
        max_length=255,
        editable=False,
    )
    officer_roles_id = models.CharField(
        max_length=255,
        editable=False,
    )
    start_date = models.DateField(
        null=True,
        editable=False,
    )
    end_date = models.DateField(
        null=True,
        editable=False,
    )
    # FKs
    human = models.ForeignKey(
        'Human',
        related_name='roles',
        editable=False,
        db_column='member_id',
        on_delete=models.CASCADE,
    )
    structure = models.ForeignKey(
        'Structure',
        related_name='roles',
        editable=False,
        db_column='object_id',
        on_delete=models.CASCADE,
    )

    created = models.DateTimeField(
        db_column='created',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='updated',
        null=True,
        editable=False,
    )

    objects = RoleManager()

    # Internals
    def __str__(self):
        return "{0} {1} {2}".format(
            self.name,
            self.human,
            self.structure,
        )

    class Meta:
        managed=False
        db_table = 'vwOfficers'

class Join(models.Model):
    id = models.CharField(
        primary_key=True,
        max_length=255,
        editable=False,
    )
    status = models.BooleanField(
        editable=False,
    )
    paid = models.BooleanField(
        editable=False,
    )
    vocal_part = models.CharField(
        max_length=255,
        editable=False,
    )
    established_date = models.DateField(
        db_column='created',
        null=False,
        editable=False,
    )
    inactive_date = models.DateField(
        db_column='inactive',
        null=True,
        editable=False,
    )
    inactive_reason = models.CharField(
        max_length=255,
        db_column='inactive_reason',
        editable=False,
    )
    created = models.DateTimeField(
        db_column='created_on',
        null=False,
        editable=False,
    )
    modified = models.DateTimeField(
        db_column='modified',
        null=True,
        editable=False,
    )

    objects = JoinManager()

    # FKs
    subscription = models.ForeignKey(
        'Subscription',
        editable=False,
        related_name='joins',
        on_delete=models.CASCADE,
    )
    membership = models.ForeignKey(
        'Membership',
        editable=False,
        related_name='joins',
        on_delete=models.CASCADE,
    )

    structure = models.ForeignKey(
        'Structure',
        editable=False,
        related_name='joins',
        db_column='reference_structure_id',
        on_delete=models.CASCADE,
    )

    # Internals
    def clean(self):
        if all([
            not self.inactive_date,
            self.subscription.items_editable,
            self.subscription.status == 'expired',
        ]):
            raise ValidationError({
                'inactive_date': 'Inactive Date is missing on expired subscription.',
            })

    def __str__(self):
        return str(self.id)
        # return "{0} {1}".format(
        #     self.subscription,
        #     self.membership,
        # )

    class Meta:
        managed = False
        db_table = 'vwSubscriptions_Memberships'
        verbose_name = 'Join'
        verbose_name_plural = 'Joins'
<|endoftext|>


<commit_before>import logging
import logging.handlers
import typing

from flask import request

from oauthclientbridge import app

if typing.TYPE_CHECKING:
    from typing import Any, List, Text, Tuple, Union  # noqa: F401

class ContextualFilter(logging.Filter):
    def filter(self, record):  # type: (Any) -> bool
        record.request_path = request.path
        record.request_base_url = request.base_url
        record.request_method = request.method
        record.request_remote_address = request.remote_addr
        return True

class CustomSMTPHandler(logging.handlers.SMTPHandler):
    def __init__(self, mailhost, fromaddr, toaddrs, subject):
        # type: (Union[str, Tuple[str, int]], str, List[str], str) -> None
        super().__init__(mailhost, fromaddr, toaddrs, subject)
        self.subject_formatter = logging.Formatter(subject)

    def getSubject(self, record):  # noqa: N802
        # type: (logging.LogRecord) -> str
        return self.subject_formatter.format(record).split("\n")[0]

context_provider = ContextualFilter()
app.logger.addFilter(context_provider)
app.logger.setLevel("DEBUG")

if app.config["OAUTH_LOG_FILE"]:
    file_handler = logging.handlers.RotatingFileHandler(
        app.config["OAUTH_LOG_FILE"],
        maxBytes=app.config["OAUTH_LOG_FILE_MAX_BYTES"],
        backupCount=app.config["OAUTH_LOG_FILE_BACKUP_COUNT"],
    )
    file_handler.setFormatter(logging.Formatter(app.config["OAUTH_LOG_FILE_FORMAT"]))
    file_handler.setLevel(app.config["OAUTH_LOG_FILE_LEVEL"])
    app.logger.addHandler(file_handler)

if not app.debug and app.config["OAUTH_LOG_EMAIL"]:
    mail_handler = CustomSMTPHandler(
        app.config["OAUTH_LOG_EMAIL_HOST"],
        app.config["OAUTH_LOG_EMAIL_FROM"],
        app.config["OAUTH_LOG_EMAIL"],
        app.config["OAUTH_LOG_EMAIL_SUBJECT"],
    )
    mail_handler.setFormatter(logging.Formatter(app.config["OAUTH_LOG_EMAIL_FORMAT"]))
    mail_handler.setLevel(app.config["OAUTH_LOG_EMAIL_LEVEL"])
    app.logger.addHandler(mail_handler)
<commit_msg>Update logging typing<commit_after>import logging
import logging.handlers
from typing import Any, List

from flask import request

from oauthclientbridge import app

class ContextualFilter(logging.Filter):
    def filter(self, record: Any) -> bool:
        record.request_path = request.path
        record.request_base_url = request.base_url
        record.request_method = request.method
        record.request_remote_address = request.remote_addr
        return True

class CustomSMTPHandler(logging.handlers.SMTPHandler):
    def __init__(self, mailhost: str, fromaddr: str, toaddrs: List[str], subject: str):
        super().__init__(mailhost, fromaddr, toaddrs, subject)
        self.subject_formatter = logging.Formatter(subject)

    def getSubject(self, record: logging.LogRecord) -> str:
        return self.subject_formatter.format(record).split("\n")[0]

context_provider = ContextualFilter()
app.logger.addFilter(context_provider)
app.logger.setLevel("DEBUG")

if app.config["OAUTH_LOG_FILE"]:
    file_handler = logging.handlers.RotatingFileHandler(
        app.config["OAUTH_LOG_FILE"],
        maxBytes=app.config["OAUTH_LOG_FILE_MAX_BYTES"],
        backupCount=app.config["OAUTH_LOG_FILE_BACKUP_COUNT"],
    )
    file_handler.setFormatter(logging.Formatter(app.config["OAUTH_LOG_FILE_FORMAT"]))
    file_handler.setLevel(app.config["OAUTH_LOG_FILE_LEVEL"])
    app.logger.addHandler(file_handler)

if not app.debug and app.config["OAUTH_LOG_EMAIL"]:
    mail_handler = CustomSMTPHandler(
        app.config["OAUTH_LOG_EMAIL_HOST"],
        app.config["OAUTH_LOG_EMAIL_FROM"],
        app.config["OAUTH_LOG_EMAIL"],
        app.config["OAUTH_LOG_EMAIL_SUBJECT"],
    )
    mail_handler.setFormatter(logging.Formatter(app.config["OAUTH_LOG_EMAIL_FORMAT"]))
    mail_handler.setLevel(app.config["OAUTH_LOG_EMAIL_LEVEL"])
    app.logger.addHandler(mail_handler)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 16 Jan 2020

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The csv_log_sync utility is used to extract JSON found from the CSV files created by the csv_logger utility.

The utility is designed to be invoked of device start-up. At this point, csv_log_sync can interrogate the server
in order to determine whether there are any found stored on the device with a more recent 'rec' datetime than the
most recent document on the server. If so, these found can be published before device sensing operations begin.

Documents are found in order of messaging topic, then datetime, with the oldest found found first.

In a typical setting, topic paths and start datetimes are found automatically from the server, an alternate test mode
is provided in which a single topic and start datetime is provided on the command line.

In order to interrogate the server, both an AWS API authentication configuration and system ID must be set. The
csv_log_sync utility is not supported by the Open Sensors data platform.

Note that the csv_log_sync utility is only able to find missing server found at the end of the device timeline -
it is not able to infill gaps within the server data timeline.

SYNOPSIS
csv_log_sync.py { -t TOPIC_NAME -s START | -f } [-n] [-w] [-p UDS_PUB] [-v]

EXAMPLES
./csv_log_sync.py -vfw -p ~/SCS/pipes/mqtt_publication.uds

SEE ALSO
scs_dev/csv_logger
scs_dev/csv_reader
scs_dev/csv_writer
scs_mfr/aws_api_auth
scs_mfr/csv_logger_conf
scs_mfr/system_id

BUGS
For compatibility with AWS DynamoDB, the --nullify flag should be used to convert empty string values to null.
"""

import sys

from scs_core.aws.client.api_auth import APIAuth
from scs_core.aws.manager.byline_manager import BylineManager

from scs_core.csv.csv_log import CSVLog
from scs_core.csv.csv_log_cursor_queue import CSVLogCursorQueue
from scs_core.csv.csv_log_reader import CSVLogReader
from scs_core.csv.csv_logger_conf import CSVLoggerConf

from scs_core.sys.signalled_exit import SignalledExit
from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_csv_log_sync import CmdCSVLogSync

from scs_host.client.http_client import HTTPClient
from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    cmd = None
    log = None
    reader = None

    try:
        # ------------------------------------------------------------------------------------------------------------
        # cmd...

        cmd = CmdCSVLogSync()

        if not cmd.is_valid():
            cmd.print_help(sys.stderr)
            exit(2)

        if cmd.verbose:
            print("csv_log_sync: %s" % cmd, file=sys.stderr)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # SystemID...
        system_id = SystemID.load(Host)

        if system_id is None:
            print("csv_log_sync: SystemID not available.", file=sys.stderr)
            exit(1)

        if cmd.verbose:
            print("csv_log_sync: %s" % system_id, file=sys.stderr)

        # CSVLoggerConf...
        conf = CSVLoggerConf.load(Host)

        if conf is None:
            print("csv_log_sync: CSVLoggerConf not available.", file=sys.stderr)
            exit(1)

        if cmd.verbose:
            print("csv_log_sync: %s" % conf, file=sys.stderr)

        if cmd.fill:
            # APIAuth...
            api_auth = APIAuth.load(Host)

            if api_auth is None:
                print("csv_log_sync: APIAuth not available.", file=sys.stderr)
                exit(1)

            # BylineManager...
            manager = BylineManager(HTTPClient(), api_auth)

            if cmd.verbose:
                print("csv_log_sync: %s" % manager, file=sys.stderr)

            # log...
            for byline in manager.find_bylines_for_device(system_id.message_tag()):
                if byline.topic_name() == cmd.topic_name:
                    log = CSVLog(conf.root_path, cmd.topic_name, system_id.message_tag(), byline.rec.datetime)
                    break

            if log is None:
                if cmd.verbose:
                    print("csv_log_sync: no backlog was found for topic %s" % cmd.topic_name, file=sys.stderr)
                exit(0)

        else:
            # log...
            log = CSVLog(conf.root_path, cmd.topic_name, system_id.message_tag(), cmd.start.datetime)

        sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("csv_log_sync", cmd.verbose)

        # read...
        queue = CSVLogCursorQueue.construct_for_log(log, 'rec')
        reader = CSVLogReader(queue, empty_string_as_null=cmd.nullify, verbose=cmd.verbose)

        reader.run(halt_on_empty_queue=True)

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except (BrokenPipeError, ConnectionResetError) as ex:
        print("csv_log_sync: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("csv_log_sync: finishing", file=sys.stderr)
<commit_msg>Updated documentation<commit_after>#!/usr/bin/env python3

"""
Created on 16 Jan 2020

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The csv_log_sync utility is used to extract JSON found from the CSV files created by the csv_logger utility.

The utility can interrogate the server in order to determine whether there are any found stored on the device with a
more recent 'rec' datetime than the most recent document on the server. If so, these found can be published before
device sensing operations begin. An alternate test mode is provided in which a single topic and start datetime is
provided on the command line.

Documents are found in order of datetime, with the oldest found found first.

In order to interrogate the server, both an AWS API authentication configuration and system ID must be set. The
csv_log_sync utility is not supported by the Open Sensors data platform.

Note that the csv_log_sync utility is only able to find missing server found at the end of the device timeline -
it is not able to infill gaps within the server data timeline.

SYNOPSIS
csv_log_sync.py -t TOPIC_NAME { -s START | -f } [-n] [-v]

EXAMPLES
./csv_log_sync.py -t climate -vf

SEE ALSO
scs_dev/csv_logger
scs_dev/csv_reader
scs_dev/csv_writer
scs_mfr/aws_api_auth
scs_mfr/csv_logger_conf
scs_mfr/system_id

BUGS
For compatibility with AWS DynamoDB, the --nullify flag should be used to convert empty string values to null.
"""

import sys

from scs_core.aws.client.api_auth import APIAuth
from scs_core.aws.manager.byline_manager import BylineManager

from scs_core.csv.csv_log import CSVLog
from scs_core.csv.csv_log_cursor_queue import CSVLogCursorQueue
from scs_core.csv.csv_log_reader import CSVLogReader
from scs_core.csv.csv_logger_conf import CSVLoggerConf

from scs_core.sys.signalled_exit import SignalledExit
from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_csv_log_sync import CmdCSVLogSync

from scs_host.client.http_client import HTTPClient
from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    cmd = None
    log = None
    reader = None

    try:
        # ------------------------------------------------------------------------------------------------------------
        # cmd...

        cmd = CmdCSVLogSync()

        if not cmd.is_valid():
            cmd.print_help(sys.stderr)
            exit(2)

        if cmd.verbose:
            print("csv_log_sync: %s" % cmd, file=sys.stderr)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # SystemID...
        system_id = SystemID.load(Host)

        if system_id is None:
            print("csv_log_sync: SystemID not available.", file=sys.stderr)
            exit(1)

        if cmd.verbose:
            print("csv_log_sync: %s" % system_id, file=sys.stderr)

        # CSVLoggerConf...
        conf = CSVLoggerConf.load(Host)

        if conf is None:
            print("csv_log_sync: CSVLoggerConf not available.", file=sys.stderr)
            exit(1)

        if cmd.verbose:
            print("csv_log_sync: %s" % conf, file=sys.stderr)

        if cmd.fill:
            # APIAuth...
            api_auth = APIAuth.load(Host)

            if api_auth is None:
                print("csv_log_sync: APIAuth not available.", file=sys.stderr)
                exit(1)

            # BylineManager...
            manager = BylineManager(HTTPClient(), api_auth)

            if cmd.verbose:
                print("csv_log_sync: %s" % manager, file=sys.stderr)

            # log...
            for byline in manager.find_bylines_for_device(system_id.message_tag()):
                if byline.topic_name() == cmd.topic_name:
                    log = CSVLog(conf.root_path, cmd.topic_name, system_id.message_tag(), byline.rec.datetime)
                    break

            if log is None:
                if cmd.verbose:
                    print("csv_log_sync: no backlog was found for topic %s" % cmd.topic_name, file=sys.stderr)
                exit(0)

        else:
            # log...
            log = CSVLog(conf.root_path, cmd.topic_name, system_id.message_tag(), cmd.start.datetime)

        sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("csv_log_sync", cmd.verbose)

        # read...
        queue = CSVLogCursorQueue.construct_for_log(log, 'rec')
        reader = CSVLogReader(queue, empty_string_as_null=cmd.nullify, verbose=cmd.verbose)

        reader.run(halt_on_empty_queue=True)

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except (BrokenPipeError, ConnectionResetError) as ex:
        print("csv_log_sync: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("csv_log_sync: finishing", file=sys.stderr)
<|endoftext|>


<commit_before>import datetime
import os
import string

from sqlalchemy import Column, Integer, String, Text, ForeignKey, DateTime
from sqlalchemy.orm import relationship

from database import Base, DeclEnum
from mod_test.nicediff import diff

class TestPlatform(DeclEnum):
    linux = "linux", "Linux"
    windows = "windows", "Windows"

class TestType(DeclEnum):
    commit = "commit", "Commit"
    pull_request = "pr", "Pull Request"

class TestStatus(DeclEnum):
    preparation = "preparation", "Preparation"
    building = "building", "Building"
    testing = "testing", "Testing"
    completed = "completed", "Completed"
    canceled = "canceled", "Canceled/Error"

    @staticmethod
    def progress_step(inst):
        try:
            return TestStatus.stages().index(inst)
        except ValueError:
            return -1

    @staticmethod
    def stages():
        return [TestStatus.preparation, TestStatus.building,
                TestStatus.testing, TestStatus.completed]

class Fork(Base):
    __tablename__ = 'fork'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    github = Column(String(256), unique=True)
    tests = relationship('Test', back_populates='fork')

    def __init__(self, github):
        self.github = github

    def __repr__(self):
        return '<Fork %r>' % self.id

    @property
    def github_url(self):
        return self.github.replace('.git', '')

    @property
    def github_name(self):
        return self.github_url.replace('https://github.com/', '')

class Test(Base):
    __tablename__ = 'test'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    platform = Column(TestPlatform.db_type(), nullable=False)
    test_type = Column(TestType.db_type(), nullable=False)
    token = Column(String(64), unique=True)
    fork_id = Column(Integer, ForeignKey('fork.id', onupdate="CASCADE",
                                         ondelete="RESTRICT"))
    fork = relationship('Fork', uselist=False, back_populates='tests')
    branch = Column(Text(), nullable=False)
    commit = Column(String(64), nullable=False)
    pr_nr = Column(Integer(), nullable=False, default=0)
    progress = relationship('TestProgress', back_populates='test',
                            order_by='TestProgress.id')
    results = relationship('TestResult', back_populates='test')

    def __init__(self, platform, test_type, fork_id, branch, commit,
                 pr_nr=0, token=None):
        self.platform = platform
        self.test_type = test_type
        self.fork_id = fork_id
        self.branch = branch
        self.commit = commit
        self.pr_nr = pr_nr
        if token is None:
            # Auto-generate token
            token = self.create_token(64)
        self.token = token

    def __repr__(self):
        return '<TestEntry %r>' % self.id

    @property
    def finished(self):
        if len(self.progress) > 0:
            return self.progress[-1].status in [
                TestStatus.completed, TestStatus.canceled]
        return False

    @property
    def failed(self):
        if len(self.progress) > 0:
            return self.progress[-1].status == TestStatus.canceled
        return False

    @property
    def github_link(self):
        if self.test_type == TestType.commit:
            test_type = 'commit'
            test_id = self.commit
        else:
            test_type = 'pull'
            test_id = self.pr_nr

        return "{base}/{test_type}/{test_id}".format(
            base=self.fork.github_url, test_type=test_type, test_id=test_id)

    def progress_data(self):
        result = {
            'progress': {
                'state': 'error',
                'step': -1
            },
            'stages': TestStatus.stages(),
            'start': '-',
            'end': '-'
        }
        if len(self.progress) > 0:
            result['start'] = self.progress[0].timestamp
            last_status = self.progress[-1]
            if last_status.status in [TestStatus.completed,
                                      TestStatus.canceled]:
                result['end'] = last_status.timestamp
            if last_status.status == TestStatus.canceled:
                if len(self.progress) > 1:
                    result['progress']['step'] = TestStatus.progress_step(
                        self.progress[-2].status)
            else:
                result['progress']['state'] = 'ok'
                result['progress']['step'] = TestStatus.progress_step(
                    last_status.status)
        return result

    @staticmethod
    def create_token(length=64):
        chars = string.ascii_letters + string.digits
        import os
        return ''.join(chars[ord(os.urandom(1)) % len(chars)] for i in
                       range(length))

class TestProgress(Base):
    __tablename__ = 'test_progress'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    test_id = Column(Integer, ForeignKey('test.id', onupdate="CASCADE",
                                         ondelete="CASCADE"))
    test = relationship('Test', uselist=False, back_populates='progress')
    status = Column(TestStatus.db_type(), nullable=False)
    timestamp = Column(DateTime(), nullable=False)
    message = Column(Text(), nullable=False)

    def __init__(self, test_id, status, message, timestamp=None):
        self.test_id = test_id
        self.status = status
        if timestamp is None:
            timestamp = datetime.datetime.now()
        self.timestamp = timestamp
        self.message = message

    def __repr__(self):
        return '<TestStatus %r: %r>' % self.test_id, self.status

class TestResult(Base):
    __tablename__ = 'test_result'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    test_id = Column(Integer, ForeignKey('test.id', onupdate="CASCADE",
                                         ondelete="CASCADE"),
                     primary_key=True)
    test = relationship('Test', uselist=False, back_populates='results')
    regression_test_id = Column(
        Integer, ForeignKey('regression_test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test = relationship('RegressionTest', uselist=False)
    runtime = Column(Integer)  # Runtime in ms
    exit_code = Column(Integer)

    def __init__(self, test_id, regression_test_id, runtime, exit_code):
        self.test_id = test_id
        self.regression_test_id = regression_test_id
        self.runtime = runtime
        self.exit_code = exit_code

    def __repr__(self):
        return '<TestResult {tid},{rid}: {code} in {time} ms>'.format(
            tid=self.test_id, rid=self.regression_test_id,
            code=self.exit_code, time=self.runtime)

class TestResultFile(Base):
    __tablename__ = 'test_result_file'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    test_id = Column(
        Integer, ForeignKey('test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    test = relationship('Test', uselist=False)
    regression_test_id = Column(
        Integer, ForeignKey('regression_test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test = relationship('RegressionTest', uselist=False)
    regression_test_output_id = Column(
        Integer, ForeignKey('regression_test_output.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test_output = relationship(
        'RegressionTestOutput', uselist=False)
    expected = Column(Text(), nullable=False)  # If the test changes in the
    # future, we need to keep track of which sample was 'correct' at the
    # time the test ran.
    got = Column(Text(), nullable=True)  # If null/empty, it's equal to the
    #  expected version

    def __init__(self, test_id, regression_test_id,
                 regression_test_output_id, expected, got=None):
        self.test_id = test_id
        self.regression_test_id = regression_test_id
        self.regression_test_output_id = regression_test_output_id
        self.expected = expected
        self.got = got

    def __repr__(self):
        return '<TestResultFile {tid},{rid},{oid}: {equal}>'.format(
            tid=self.test_id, rid=self.regression_test_id,
            oid=self.regression_test_output_id,
            equal="Equal" if self.got is None else "Unequal"
        )

    def generate_html_diff(self, base_path):
        # TODO: use difflib.SequenceMatcher and generate own HTML diff
        file_ok = os.path.join(
            base_path,
            self.expected + self.regression_test_output.correct_extension)
        file_fail = os.path.join(
            base_path,
            self.got + self.regression_test_output.correct_extension)
        lines_ok = open(file_ok, 'U').readlines()
        lines_fail = open(file_fail, 'U').readlines()

        return diff.get_html_diff(lines_ok, lines_fail)
<commit_msg>Adding timezone in naive datetime.<commit_after>import datetime
import pytz
import tzlocal
import os
import string

from sqlalchemy import Column, Integer, String, Text, ForeignKey, DateTime
from sqlalchemy.orm import relationship

from database import Base, DeclEnum
from mod_test.nicediff import diff

class TestPlatform(DeclEnum):
    linux = "linux", "Linux"
    windows = "windows", "Windows"

class TestType(DeclEnum):
    commit = "commit", "Commit"
    pull_request = "pr", "Pull Request"

class TestStatus(DeclEnum):
    preparation = "preparation", "Preparation"
    building = "building", "Building"
    testing = "testing", "Testing"
    completed = "completed", "Completed"
    canceled = "canceled", "Canceled/Error"

    @staticmethod
    def progress_step(inst):
        try:
            return TestStatus.stages().index(inst)
        except ValueError:
            return -1

    @staticmethod
    def stages():
        return [TestStatus.preparation, TestStatus.building,
                TestStatus.testing, TestStatus.completed]

class Fork(Base):
    __tablename__ = 'fork'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    github = Column(String(256), unique=True)
    tests = relationship('Test', back_populates='fork')

    def __init__(self, github):
        self.github = github

    def __repr__(self):
        return '<Fork %r>' % self.id

    @property
    def github_url(self):
        return self.github.replace('.git', '')

    @property
    def github_name(self):
        return self.github_url.replace('https://github.com/', '')

class Test(Base):
    __tablename__ = 'test'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    platform = Column(TestPlatform.db_type(), nullable=False)
    test_type = Column(TestType.db_type(), nullable=False)
    token = Column(String(64), unique=True)
    fork_id = Column(Integer, ForeignKey('fork.id', onupdate="CASCADE",
                                         ondelete="RESTRICT"))
    fork = relationship('Fork', uselist=False, back_populates='tests')
    branch = Column(Text(), nullable=False)
    commit = Column(String(64), nullable=False)
    pr_nr = Column(Integer(), nullable=False, default=0)
    progress = relationship('TestProgress', back_populates='test',
                            order_by='TestProgress.id')
    results = relationship('TestResult', back_populates='test')

    def __init__(self, platform, test_type, fork_id, branch, commit,
                 pr_nr=0, token=None):
        self.platform = platform
        self.test_type = test_type
        self.fork_id = fork_id
        self.branch = branch
        self.commit = commit
        self.pr_nr = pr_nr
        if token is None:
            # Auto-generate token
            token = self.create_token(64)
        self.token = token

    def __repr__(self):
        return '<TestEntry %r>' % self.id

    @property
    def finished(self):
        if len(self.progress) > 0:
            return self.progress[-1].status in [
                TestStatus.completed, TestStatus.canceled]
        return False

    @property
    def failed(self):
        if len(self.progress) > 0:
            return self.progress[-1].status == TestStatus.canceled
        return False

    @property
    def github_link(self):
        if self.test_type == TestType.commit:
            test_type = 'commit'
            test_id = self.commit
        else:
            test_type = 'pull'
            test_id = self.pr_nr

        return "{base}/{test_type}/{test_id}".format(
            base=self.fork.github_url, test_type=test_type, test_id=test_id)

    def progress_data(self):
        result = {
            'progress': {
                'state': 'error',
                'step': -1
            },
            'stages': TestStatus.stages(),
            'start': '-',
            'end': '-'
        }
        if len(self.progress) > 0:
            result['start'] = self.progress[0].timestamp
            last_status = self.progress[-1]
            if last_status.status in [TestStatus.completed,
                                      TestStatus.canceled]:
                result['end'] = last_status.timestamp
            if last_status.status == TestStatus.canceled:
                if len(self.progress) > 1:
                    result['progress']['step'] = TestStatus.progress_step(
                        self.progress[-2].status)
            else:
                result['progress']['state'] = 'ok'
                result['progress']['step'] = TestStatus.progress_step(
                    last_status.status)
        return result

    @staticmethod
    def create_token(length=64):
        chars = string.ascii_letters + string.digits
        import os
        return ''.join(chars[ord(os.urandom(1)) % len(chars)] for i in
                       range(length))

class TestProgress(Base):
    __tablename__ = 'test_progress'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    id = Column(Integer, primary_key=True)
    test_id = Column(Integer, ForeignKey('test.id', onupdate="CASCADE",
                                         ondelete="CASCADE"))
    test = relationship('Test', uselist=False, back_populates='progress')
    status = Column(TestStatus.db_type(), nullable=False)
    timestamp = Column(DateTime(), nullable=False)
    message = Column(Text(), nullable=False)

    def __init__(self, test_id, status, message, timestamp=None):
        self.test_id = test_id
        self.status = status
        tz=get_localzone()
        if timestamp is None:
            timestamp = tz.localize(datetime.datetime.now(),is_dst=None)
        self.timestamp = timestamp.astimezone(pytz.UTC) #Storing timestamp in UTC
        self.message = message

    def __repr__(self):
        return '<TestStatus %r: %r>' % self.test_id, self.status

class TestResult(Base):
    __tablename__ = 'test_result'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    test_id = Column(Integer, ForeignKey('test.id', onupdate="CASCADE",
                                         ondelete="CASCADE"),
                     primary_key=True)
    test = relationship('Test', uselist=False, back_populates='results')
    regression_test_id = Column(
        Integer, ForeignKey('regression_test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test = relationship('RegressionTest', uselist=False)
    runtime = Column(Integer)  # Runtime in ms
    exit_code = Column(Integer)

    def __init__(self, test_id, regression_test_id, runtime, exit_code):
        self.test_id = test_id
        self.regression_test_id = regression_test_id
        self.runtime = runtime
        self.exit_code = exit_code

    def __repr__(self):
        return '<TestResult {tid},{rid}: {code} in {time} ms>'.format(
            tid=self.test_id, rid=self.regression_test_id,
            code=self.exit_code, time=self.runtime)

class TestResultFile(Base):
    __tablename__ = 'test_result_file'
    __table_args__ = {'mysql_engine': 'InnoDB'}
    test_id = Column(
        Integer, ForeignKey('test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    test = relationship('Test', uselist=False)
    regression_test_id = Column(
        Integer, ForeignKey('regression_test.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test = relationship('RegressionTest', uselist=False)
    regression_test_output_id = Column(
        Integer, ForeignKey('regression_test_output.id', onupdate="CASCADE",
                            ondelete="CASCADE"), primary_key=True
    )
    regression_test_output = relationship(
        'RegressionTestOutput', uselist=False)
    expected = Column(Text(), nullable=False)  # If the test changes in the
    # future, we need to keep track of which sample was 'correct' at the
    # time the test ran.
    got = Column(Text(), nullable=True)  # If null/empty, it's equal to the
    #  expected version

    def __init__(self, test_id, regression_test_id,
                 regression_test_output_id, expected, got=None):
        self.test_id = test_id
        self.regression_test_id = regression_test_id
        self.regression_test_output_id = regression_test_output_id
        self.expected = expected
        self.got = got

    def __repr__(self):
        return '<TestResultFile {tid},{rid},{oid}: {equal}>'.format(
            tid=self.test_id, rid=self.regression_test_id,
            oid=self.regression_test_output_id,
            equal="Equal" if self.got is None else "Unequal"
        )

    def generate_html_diff(self, base_path):
        # TODO: use difflib.SequenceMatcher and generate own HTML diff
        file_ok = os.path.join(
            base_path,
            self.expected + self.regression_test_output.correct_extension)
        file_fail = os.path.join(
            base_path,
            self.got + self.regression_test_output.correct_extension)
        lines_ok = open(file_ok, 'U').readlines()
        lines_fail = open(file_fail, 'U').readlines()

        return diff.get_html_diff(lines_ok, lines_fail)
<|endoftext|>


<commit_before>#    Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import os
import testtools
import pexpect
from mock import Mock, MagicMock
from trove.guestagent import volume
from trove.common import utils

def _setUp_fake_spawn(return_val=0):
    fake_spawn = pexpect.spawn('echo')
    fake_spawn.expect = Mock(return_value=return_val)
    pexpect.spawn = Mock(return_value=fake_spawn)
    return fake_spawn

class VolumeDeviceTest(testtools.TestCase):

    def setUp(self):
        super(VolumeDeviceTest, self).setUp()
        self.volumeDevice = volume.VolumeDevice('/dev/vdb')

    def tearDown(self):
        super(VolumeDeviceTest, self).tearDown()

    def test_migrate_data(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()
        fake_spawn = _setUp_fake_spawn()

        origin_unmount = self.volumeDevice.unmount
        self.volumeDevice.unmount = MagicMock()
        self.volumeDevice.migrate_data('/')
        self.assertEqual(1, fake_spawn.expect.call_count)
        self.assertEqual(1, utils.execute.call_count)
        self.assertEqual(1, self.volumeDevice.unmount.call_count)
        utils.execute = origin_execute
        self.volumeDevice.unmount = origin_unmount
        os.path.exists = origin_os_path_exists

    def test__check_device_exists(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        self.volumeDevice._check_device_exists()
        self.assertEqual(1, utils.execute.call_count)
        utils.execute = origin_execute

    def test__check_format(self):
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice._check_format()
        self.assertEqual(1, fake_spawn.expect.call_count)

    def test__check_format_2(self):
        fake_spawn = _setUp_fake_spawn(return_val=1)

        self.assertEqual(0, fake_spawn.expect.call_count)
        self.assertRaises(IOError, self.volumeDevice._check_format)

    def test__format(self):
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice._format()

        self.assertEqual(1, fake_spawn.expect.call_count)
        self.assertEqual(1, pexpect.spawn.call_count)

    def test_format(self):
        origin_check_device_exists = self.volumeDevice._check_device_exists
        origin_format = self.volumeDevice._format
        origin_check_format = self.volumeDevice._check_format
        self.volumeDevice._check_device_exists = MagicMock()
        self.volumeDevice._check_format = MagicMock()
        self.volumeDevice._format = MagicMock()

        self.volumeDevice.format()
        self.assertEqual(1, self.volumeDevice._check_device_exists.call_count)
        self.assertEqual(1, self.volumeDevice._format.call_count)
        self.assertEqual(1, self.volumeDevice._check_format.call_count)

        self.volumeDevice._check_device_exists = origin_check_device_exists
        self.volumeDevice._format = origin_format
        self.volumeDevice._check_format = origin_check_format

    def test_mount(self):
        origin_ = volume.VolumeMountPoint.mount
        volume.VolumeMountPoint.mount = Mock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()
        origin_write_to_fstab = volume.VolumeMountPoint.write_to_fstab
        volume.VolumeMountPoint.write_to_fstab = Mock()

        self.volumeDevice.mount(Mock)
        self.assertEqual(1, volume.VolumeMountPoint.mount.call_count)
        self.assertEqual(1, volume.VolumeMountPoint.write_to_fstab.call_count)
        volume.VolumeMountPoint.mount = origin_
        volume.VolumeMountPoint.write_to_fstab = origin_write_to_fstab
        os.path.exists = origin_os_path_exists

    def test_resize_fs(self):
        origin_check_device_exists = self.volumeDevice._check_device_exists
        origin_execute = utils.execute
        utils.execute = Mock()
        self.volumeDevice._check_device_exists = MagicMock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()

        self.volumeDevice.resize_fs('/mnt/volume')

        self.assertEqual(1, self.volumeDevice._check_device_exists.call_count)
        self.assertEqual(2, utils.execute.call_count)
        self.volumeDevice._check_device_exists = origin_check_device_exists
        os.path.exists = origin_os_path_exists
        utils.execute = origin_execute

    def test_unmount_positive(self):
        self._test_unmount()

    def test_unmount_negative(self):
        self._test_unmount(False)

    def _test_unmount(self, positive=True):
        origin_ = os.path.exists
        os.path.exists = MagicMock(return_value=positive)
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice.unmount('/mnt/volume')
        COUNT = 1
        if not positive:
            COUNT = 0
        self.assertEqual(COUNT, fake_spawn.expect.call_count)
        os.path.exists = origin_

class VolumeMountPointTest(testtools.TestCase):
    def setUp(self):
        super(VolumeMountPointTest, self).setUp()
        self.volumeMountPoint = volume.VolumeMountPoint('/mnt/device',
                                                        '/dev/vdb')

    def tearDown(self):
        super(VolumeMountPointTest, self).tearDown()

    def test_mount(self):
        origin_ = os.path.exists
        os.path.exists = MagicMock(return_value=False)
        fake_spawn = _setUp_fake_spawn()

        utils.execute = Mock()

        self.volumeMountPoint.mount()

        self.assertEqual(1, os.path.exists.call_count)
        self.assertEqual(1, utils.execute.call_count)
        self.assertEqual(1, fake_spawn.expect.call_count)

        os.path.exists = origin_

    def test_write_to_fstab(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        open = MagicMock()
        # Avoiding error at PEP8 F841 rule
        if open:
            pass
        self.volumeMountPoint.write_to_fstab()

        self.assertEqual(2, utils.execute.call_count)
        utils.execute = origin_execute
<commit_msg>Mocks out file.open to not rely on OS dependent files<commit_after>#    Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import os
import testtools
import pexpect
from mock import Mock, MagicMock, patch, mock_open
from trove.guestagent import volume
from trove.common import utils

def _setUp_fake_spawn(return_val=0):
    fake_spawn = pexpect.spawn('echo')
    fake_spawn.expect = Mock(return_value=return_val)
    pexpect.spawn = Mock(return_value=fake_spawn)
    return fake_spawn

class VolumeDeviceTest(testtools.TestCase):

    def setUp(self):
        super(VolumeDeviceTest, self).setUp()
        self.volumeDevice = volume.VolumeDevice('/dev/vdb')

    def tearDown(self):
        super(VolumeDeviceTest, self).tearDown()

    def test_migrate_data(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()
        fake_spawn = _setUp_fake_spawn()

        origin_unmount = self.volumeDevice.unmount
        self.volumeDevice.unmount = MagicMock()
        self.volumeDevice.migrate_data('/')
        self.assertEqual(1, fake_spawn.expect.call_count)
        self.assertEqual(1, utils.execute.call_count)
        self.assertEqual(1, self.volumeDevice.unmount.call_count)
        utils.execute = origin_execute
        self.volumeDevice.unmount = origin_unmount
        os.path.exists = origin_os_path_exists

    def test__check_device_exists(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        self.volumeDevice._check_device_exists()
        self.assertEqual(1, utils.execute.call_count)
        utils.execute = origin_execute

    def test__check_format(self):
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice._check_format()
        self.assertEqual(1, fake_spawn.expect.call_count)

    def test__check_format_2(self):
        fake_spawn = _setUp_fake_spawn(return_val=1)

        self.assertEqual(0, fake_spawn.expect.call_count)
        self.assertRaises(IOError, self.volumeDevice._check_format)

    def test__format(self):
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice._format()

        self.assertEqual(1, fake_spawn.expect.call_count)
        self.assertEqual(1, pexpect.spawn.call_count)

    def test_format(self):
        origin_check_device_exists = self.volumeDevice._check_device_exists
        origin_format = self.volumeDevice._format
        origin_check_format = self.volumeDevice._check_format
        self.volumeDevice._check_device_exists = MagicMock()
        self.volumeDevice._check_format = MagicMock()
        self.volumeDevice._format = MagicMock()

        self.volumeDevice.format()
        self.assertEqual(1, self.volumeDevice._check_device_exists.call_count)
        self.assertEqual(1, self.volumeDevice._format.call_count)
        self.assertEqual(1, self.volumeDevice._check_format.call_count)

        self.volumeDevice._check_device_exists = origin_check_device_exists
        self.volumeDevice._format = origin_format
        self.volumeDevice._check_format = origin_check_format

    def test_mount(self):
        origin_ = volume.VolumeMountPoint.mount
        volume.VolumeMountPoint.mount = Mock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()
        origin_write_to_fstab = volume.VolumeMountPoint.write_to_fstab
        volume.VolumeMountPoint.write_to_fstab = Mock()

        self.volumeDevice.mount(Mock)
        self.assertEqual(1, volume.VolumeMountPoint.mount.call_count)
        self.assertEqual(1, volume.VolumeMountPoint.write_to_fstab.call_count)
        volume.VolumeMountPoint.mount = origin_
        volume.VolumeMountPoint.write_to_fstab = origin_write_to_fstab
        os.path.exists = origin_os_path_exists

    def test_resize_fs(self):
        origin_check_device_exists = self.volumeDevice._check_device_exists
        origin_execute = utils.execute
        utils.execute = Mock()
        self.volumeDevice._check_device_exists = MagicMock()
        origin_os_path_exists = os.path.exists
        os.path.exists = Mock()

        self.volumeDevice.resize_fs('/mnt/volume')

        self.assertEqual(1, self.volumeDevice._check_device_exists.call_count)
        self.assertEqual(2, utils.execute.call_count)
        self.volumeDevice._check_device_exists = origin_check_device_exists
        os.path.exists = origin_os_path_exists
        utils.execute = origin_execute

    def test_unmount_positive(self):
        self._test_unmount()

    def test_unmount_negative(self):
        self._test_unmount(False)

    def _test_unmount(self, positive=True):
        origin_ = os.path.exists
        os.path.exists = MagicMock(return_value=positive)
        fake_spawn = _setUp_fake_spawn()

        self.volumeDevice.unmount('/mnt/volume')
        COUNT = 1
        if not positive:
            COUNT = 0
        self.assertEqual(COUNT, fake_spawn.expect.call_count)
        os.path.exists = origin_

class VolumeMountPointTest(testtools.TestCase):
    def setUp(self):
        super(VolumeMountPointTest, self).setUp()
        self.volumeMountPoint = volume.VolumeMountPoint('/mnt/device',
                                                        '/dev/vdb')

    def tearDown(self):
        super(VolumeMountPointTest, self).tearDown()

    def test_mount(self):
        origin_ = os.path.exists
        os.path.exists = MagicMock(return_value=False)
        fake_spawn = _setUp_fake_spawn()

        utils.execute = Mock()

        self.volumeMountPoint.mount()

        self.assertEqual(1, os.path.exists.call_count)
        self.assertEqual(1, utils.execute.call_count)
        self.assertEqual(1, fake_spawn.expect.call_count)

        os.path.exists = origin_

    def test_write_to_fstab(self):
        origin_execute = utils.execute
        utils.execute = Mock()
        m = mock_open()
        with patch('%s.open' % volume.__name__, m, create=True):
            self.volumeMountPoint.write_to_fstab()

        self.assertEqual(2, utils.execute.call_count)
        utils.execute = origin_execute
<|endoftext|>


<commit_before>import os
import csv
import logging
from os.path import abspath, dirname, join
import rdflib
from indra.util import read_unicode_csv, write_unicode_csv

logger = logging.getLogger('go_client')

go_mappings_file = join(dirname(abspath(__file__)), '..', 'resources',
                 'go_id_label_mappings.tsv')

# This file can be donwloaded from: http://geneontology.org/ontology/go.owl
go_owl_path = join(dirname(abspath(__file__)), '..', '..', 'data', 'go.owl')

# Dictionary to store GO ID->Label mappings
go_mappings = {}
for go_id, go_label in read_unicode_csv(go_mappings_file, delimiter='\t'):
    go_mappings[go_id] = go_label

_prefixes = """
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    PREFIX go: <http://purl.obolibrary.org/obo/go#>
    PREFIX obo: <http://purl.obolibrary.org/obo/>
    PREFIX owl: <http://www.w3.org/2002/07/owl#>
    PREFIX oboInOwl: <http://www.geneontology.org/formats/oboInOwl#>
    PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
    """

# Lazily initialize the GO RDF graph because parsing the RDF is expensive
_go_graph = None

def load_go_graph(go_fname):
    global _go_graph
    if _go_graph is None:
        _go_graph = rdflib.Graph()
        logger.info("Parsing GO OWL file")
        _go_graph.parse(os.path.abspath(go_fname))
    return _go_graph

def get_go_label(go_id):
    return go_mappings.get(go_id)

def update_id_mappings(g):
    """Compile all ID->label mappings from the GO OWL file."""
    g = load_go_graph(go_owl_path)

    query = _prefixes + """
        SELECT ?id ?label
        WHERE {
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label
        }
    """
    logger.info("Querying for GO ID mappings")
    res = g.query(query)
    mappings = []
    for id_lit, label_lit in res:
        mappings.append((id_lit.value, label_lit.value))
    # Write to file
    write_unicode_csv(go_mappings_file, mappings)

def get_cellular_components(g):
    # Query for direct part_of relationships
    query = _prefixes + """
        SELECT ?id ?label ?supid ?suplabel
        WHERE {
            ?class oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label .
            ?class rdfs:subClassOf ?sup .
            ?sup oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?sup oboInOwl:id ?supid .
            ?sup rdfs:label ?suplabel
            }
        """
    logger.info("Running cellular component query 1")
    res1 = g.query(query)
    query = _prefixes + """
        SELECT ?id ?label ?supid ?suplabel
        WHERE {
            ?class oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label .
            ?class rdfs:subClassOf ?restr .
            ?restr owl:onProperty ?prop .
            ?prop oboInOwl:id "part_of"^^xsd:string .
            ?restr owl:someValuesFrom ?sup .
            ?sup oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?sup oboInOwl:id ?supid .
            ?sup rdfs:label ?suplabel
            }
        """
    logger.info("Running cellular component query 2")
    res2 = g.query(query)
    res = list(res1) + list(res2)
    component_map = {}
    component_part_map = {}
    for r in res:
        comp_id, comp_name, sup_id, sup_name = [rr.toPython() for rr in r]
        component_map[comp_id] = comp_name
        component_map[sup_id] = sup_name
        try:
            component_part_map[comp_id].append(sup_id)
        except KeyError:
            component_part_map[comp_id] = [sup_id]
    return component_map, component_part_map

<commit_msg>Add docs, sort mappings file by ID<commit_after>import os
import csv
import logging
from os.path import abspath, dirname, join
import rdflib
from indra.util import read_unicode_csv, write_unicode_csv

logger = logging.getLogger('go_client')

go_mappings_file = join(dirname(abspath(__file__)), '..', 'resources',
                 'go_id_label_mappings.tsv')

# This file can be donwloaded from: http://geneontology.org/ontology/go.owl
go_owl_path = join(dirname(abspath(__file__)), '..', '..', 'data', 'go.owl')

# Dictionary to store GO ID->Label mappings
go_mappings = {}
for go_id, go_label in read_unicode_csv(go_mappings_file, delimiter='\t'):
    go_mappings[go_id] = go_label

_prefixes = """
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    PREFIX go: <http://purl.obolibrary.org/obo/go#>
    PREFIX obo: <http://purl.obolibrary.org/obo/>
    PREFIX owl: <http://www.w3.org/2002/07/owl#>
    PREFIX oboInOwl: <http://www.geneontology.org/formats/oboInOwl#>
    PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
    """

# Lazily initialize the GO RDF graph because parsing the RDF is expensive
_go_graph = None

def load_go_graph(go_fname):
    """Load the GO data from an OWL file and parse into an RDF graph.

    Parameters
    ----------
    go_fname : str
        Path to the GO OWL file. Can be downloaded from
        http://geneontology.org/ontology/go.owl.

    Returns
    -------
    rdflib.Graph
        RDF graph containing GO data.
    """
    global _go_graph
    if _go_graph is None:
        _go_graph = rdflib.Graph()
        logger.info("Parsing GO OWL file")
        _go_graph.parse(os.path.abspath(go_fname))
    return _go_graph

def get_go_label(go_id):
    """Get label corresponding to a given GO identifier.

    Parameters
    ----------
    go_id : str
        The GO identifier. Should include the `GO:` prefix, e.g., `GO:1903793`
        (positive regulation of anion transport).

    Returns
    -------
    str
        Label corresponding to the GO ID.
    """
    return go_mappings.get(go_id)

def update_id_mappings(g):
    """Compile all ID->label mappings and save to a TSV file.

    Parameters
    ----------
    g : rdflib.Graph
        RDF graph containing GO data.
    """
    g = load_go_graph(go_owl_path)

    query = _prefixes + """
        SELECT ?id ?label
        WHERE {
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label
        }
    """
    logger.info("Querying for GO ID mappings")
    res = g.query(query)
    mappings = []
    for id_lit, label_lit in sorted(res, key=lambda x: x[0]):
        mappings.append((id_lit.value, label_lit.value))
    # Write to file
    write_unicode_csv(go_mappings_file, mappings)

def get_cellular_components(g):
    # Query for direct part_of relationships
    query = _prefixes + """
        SELECT ?id ?label ?supid ?suplabel
        WHERE {
            ?class oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label .
            ?class rdfs:subClassOf ?sup .
            ?sup oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?sup oboInOwl:id ?supid .
            ?sup rdfs:label ?suplabel
            }
        """
    logger.info("Running cellular component query 1")
    res1 = g.query(query)
    query = _prefixes + """
        SELECT ?id ?label ?supid ?suplabel
        WHERE {
            ?class oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?class oboInOwl:id ?id .
            ?class rdfs:label ?label .
            ?class rdfs:subClassOf ?restr .
            ?restr owl:onProperty ?prop .
            ?prop oboInOwl:id "part_of"^^xsd:string .
            ?restr owl:someValuesFrom ?sup .
            ?sup oboInOwl:hasOBONamespace "cellular_component"^^xsd:string .
            ?sup oboInOwl:id ?supid .
            ?sup rdfs:label ?suplabel
            }
        """
    logger.info("Running cellular component query 2")
    res2 = g.query(query)
    res = list(res1) + list(res2)
    component_map = {}
    component_part_map = {}
    for r in res:
        comp_id, comp_name, sup_id, sup_name = [rr.toPython() for rr in r]
        component_map[comp_id] = comp_name
        component_map[sup_id] = sup_name
        try:
            component_part_map[comp_id].append(sup_id)
        except KeyError:
            component_part_map[comp_id] = [sup_id]
    return component_map, component_part_map

<|endoftext|>


<commit_before>import xml.etree.ElementTree as ET

def xml_parse(f):
    tree = ET.parse(f)
    xmlroot = tree.getroot()
    size = xmlroot.find('size')
    width = int(size.find('width').text)
    height = int(size.find('height').text)

    ground_truths = []
    for obj in xmlroot.iter('object'):
        xmlbox = obj.find('bndbox')
        bbox = {
            'xmin': int(xmlbox.find('xmin').text),
            'ymin': int(xmlbox.find('ymin').text),
            'xmax': int(xmlbox.find('xmax').text),
            'ymax': int(xmlbox.find('ymax').text),
            'label': obj.find('name').text
        }
        try:
            bbox['value'] = str(xmlbox.find('value').text)
        except:
            pass
        finally:
            ground_truths.append(bbox)
    return (width, height), ground_truths

def voc_load(file_path):
    with open(file_path) as f:
        ret = xml_parse(f)
    return ret
<commit_msg>[update]: strip str.<commit_after>import xml.etree.ElementTree as ET

def xml_parse(f):
    tree = ET.parse(f)
    xmlroot = tree.getroot()
    size = xmlroot.find('size')
    width = int(size.find('width').text)
    height = int(size.find('height').text)

    ground_truths = []
    for obj in xmlroot.iter('object'):
        xmlbox = obj.find('bndbox')
        bbox = {
            'xmin': int(xmlbox.find('xmin').text),
            'ymin': int(xmlbox.find('ymin').text),
            'xmax': int(xmlbox.find('xmax').text),
            'ymax': int(xmlbox.find('ymax').text),
            'label': obj.find('name').text.strip()
        }
        try:
            bbox['value'] = str(xmlbox.find('value').text)
        except:
            pass
        finally:
            ground_truths.append(bbox)
    return (width, height), ground_truths

def voc_load(file_path):
    with open(file_path) as f:
        ret = xml_parse(f)
    return ret
<|endoftext|>