<commit_before># -*- coding: utf-8 -*-

import httplib
import logging

from django.db import transaction, connection
from django_bulk_update.helper import bulk_update
from django.contrib.contenttypes.models import ContentType

from addons.osfstorage.models import OsfStorageFile
from framework.auth import get_or_create_user
from framework.exceptions import HTTPError
from framework.flask import redirect
from framework.transactions.handlers import no_auto_transaction
from osf.models import AbstractNode, Node, Conference, Tag, OSFUser
from website import settings
from website.conferences import utils, signals
from website.conferences.message import ConferenceMessage, ConferenceError
from website.ember_osf_web.decorators import ember_flag_is_active
from website.mails import CONFERENCE_SUBMITTED, CONFERENCE_INACTIVE, CONFERENCE_FAILED
from website.mails import send_mail
from website.util import web_url_for

logger = logging.getLogger(__name__)

@no_auto_transaction
def meeting_hook():
    """View function for email conference submission.
    """
    message = ConferenceMessage()

    try:
        message.verify()
    except ConferenceError as error:
        logger.error(error)
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    try:
        conference = Conference.get_by_endpoint(message.conference_name, active=False)
    except ConferenceError as error:
        logger.error(error)
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    if not conference.active:
        send_mail(
            message.sender_email,
            CONFERENCE_INACTIVE,
            fullname=message.sender_display,
            presentations_url=web_url_for('conference_view', _absolute=True),
        )
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    add_poster_by_email(conference=conference, message=message)

def add_poster_by_email(conference, message):
    """
    :param Conference conference:
    :param ConferenceMessage message:
    """
    # Fail if no attachments
    if not message.attachments:
        return send_mail(
            message.sender_email,
            CONFERENCE_FAILED,
            fullname=message.sender_display,
        )

    nodes_created = []
    users_created = []

    with transaction.atomic():
        user, user_created = get_or_create_user(
            message.sender_display,
            message.sender_email,
            is_spam=message.is_spam,
        )
        if user_created:
            user.save()  # need to save in order to access m2m fields (e.g. tags)
            users_created.append(user)
            user.add_system_tag('osf4m')
            user.update_date_last_login()
            user.save()

            # must save the user first before accessing user._id
            set_password_url = web_url_for(
                'reset_password_get',
                uid=user._id,
                token=user.verification_key_v2['token'],
                _absolute=True,
            )
        else:
            set_password_url = None

        node, node_created = Node.objects.get_or_create(
            title__iexact=message.subject,
            is_deleted=False,
            _contributors__guids___id=user._id,
            defaults={
                'title': message.subject,
                'creator': user
            }
        )
        if node_created:
            nodes_created.append(node)
            node.add_system_tag('osf4m')
            node.save()

        utils.provision_node(conference, message, node, user)
        utils.record_message(message, nodes_created, users_created)
    # Prevent circular import error
    from framework.auth import signals as auth_signals
    if user_created:
        auth_signals.user_confirmed.send(user)

    utils.upload_attachments(user, node, message.attachments)

    download_url = node.web_url_for(
        'addon_view_or_download_file',
        path=message.attachments[0].filename,
        provider='osfstorage',
        action='download',
        _absolute=True,
    )

    # Send confirmation email
    send_mail(
        message.sender_email,
        CONFERENCE_SUBMITTED,
        conf_full_name=conference.name,
        conf_view_url=web_url_for(
            'conference_results',
            meeting=message.conference_name,
            _absolute=True,
        ),
        fullname=message.sender_display,
        user_created=user_created,
        set_password_url=set_password_url,
        profile_url=user.absolute_url,
        node_url=node.absolute_url,
        file_url=download_url,
        presentation_type=message.conference_category.lower(),
        is_spam=message.is_spam,
    )
    if node_created and user_created:
        signals.osf4m_user_created.send(user, conference=conference, node=node)

## TODO DELETE
def _render_conference_node(node, idx, conf):
    record = OsfStorageFile.objects.filter(node=node).first()

    if not record:
        download_url = ''
        download_count = 0
    else:
        download_count = record.get_download_count()
        download_url = node.web_url_for(
            'addon_view_or_download_file',
            path=record.path.strip('/'),
            provider='osfstorage',
            action='download',
            _absolute=True,
        )

    author = node.visible_contributors[0]
    tags = list(node.tags.filter(system=False).values_list('name', flat=True))

    return {
        'id': idx,
        'title': node.title,
        'nodeUrl': node.url,
        'author': author.family_name if author.family_name else author.fullname,
        'authorUrl': author.url,
        'category': conf.field_names['submission1'] if conf.field_names['submission1'] in tags else conf.field_names['submission2'],
        'download': download_count,
        'downloadUrl': download_url,
        'dateCreated': node.created.isoformat(),
        'confName': conf.name,
        'confUrl': web_url_for('conference_results', meeting=conf.endpoint),
        'tags': ' '.join(tags)
    }

def conference_data(meeting):
    try:
        conf = Conference.objects.get(endpoint__iexact=meeting)
    except Conference.DoesNotExist:
        raise HTTPError(httplib.NOT_FOUND)

    return conference_submissions_sql(conf)

def conference_submissions_sql(conf):
    submission1_name = conf.field_names['submission1']
    submission2_name = conf.field_names['submission2']
    conference_url = web_url_for('conference_results', meeting=conf.endpoint)
    abstract_node_content_type_id = ContentType.objects.get_for_model(AbstractNode).id
    osf_user_content_type_id = ContentType.objects.get_for_model(OSFUser).id

    with connection.cursor() as cursor:
        cursor.execute(
            """
            SELECT  json_build_object(
                    'id', osf_abstractnode.id,
                    'title', osf_abstractnode.title,
                    'nodeUrl', '/' || guid._id || '/',
                    'author', CASE WHEN AUTHOR.family_name != '' THEN AUTHOR.family_name ELSE AUTHOR.fullname END,
                    'authorUrl', '/' || author_guid._id || '/',
                    'category', COALESCE(meeting_category.name, %s),
                    'download', COALESCE(DOWNLOAD_COUNT, 0),
                    'downloadUrl', COALESCE('/project/' || guid._id || '/files/osfstorage/' || FILE._id || '/?action=download', ''),
                    'dateCreated', osf_abstractnode.created,
                    'confName', %s,
                    'confUrl', %s,
                    'tags', tags_list.tag_list
                )
            FROM osf_abstractnode
              INNER JOIN "osf_abstractnode_tags" ON ("osf_abstractnode"."id" = "osf_abstractnode_tags"."abstractnode_id")
              LEFT JOIN (
                SELECT osf_tag.name as name, osf_abstractnode_tags.abstractnode_id as node_id
                FROM osf_tag
                INNER JOIN osf_abstractnode_tags ON (osf_tag.id = osf_abstractnode_tags.tag_id)
                WHERE (osf_tag.system = false
                       AND osf_tag.name = %s)
              ) AS meeting_category ON (osf_abstractnode.id = meeting_category.node_id)
              LEFT JOIN (
                SELECT osf_abstractnode_tags.abstractnode_id, string_agg(osf_tag.name, ' ') AS tag_list
                FROM osf_tag
                RIGHT JOIN osf_abstractnode_tags ON (osf_tag.id = osf_abstractnode_tags.tag_id)
                GROUP BY osf_abstractnode_tags.abstractnode_id
              ) AS tags_list ON (osf_abstractnode.id = tags_list.abstractnode_id)
              LEFT JOIN LATERAL (
                        SELECT osf_osfuser.*
                        FROM osf_osfuser
                          INNER JOIN osf_contributor ON (osf_contributor.user_id = osf_osfuser.id)
                        WHERE (osf_contributor.node_id = osf_abstractnode.id AND osf_contributor.visible = TRUE)
                        LIMIT 1
                        ) AUTHOR ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_guid._id
                FROM osf_guid
                WHERE (osf_guid.object_id = osf_abstractnode.id AND osf_guid.content_type_id = %s) -- Content type for AbstractNode
                ORDER BY osf_guid.created DESC
                LIMIT 1
              ) GUID ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_guid._id
                FROM osf_guid
                WHERE (osf_guid.object_id = AUTHOR.id AND osf_guid.content_type_id = %s)  -- Content type for OSFUser
                LIMIT 1
              ) AUTHOR_GUID ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_basefilenode.*
                FROM osf_basefilenode
                WHERE (
                  osf_basefilenode.type = 'osf.osfstoragefile'
                  AND osf_basefilenode.provider = 'osfstorage'
                  AND osf_basefilenode.node_id = osf_abstractnode.id
                )
                LIMIT 1
              ) FILE ON TRUE
              LEFT JOIN LATERAL (
                SELECT P.total AS DOWNLOAD_COUNT
                FROM osf_pagecounter AS P
                WHERE P._id = 'download:' || GUID._id || ':' || FILE._id
                LIMIT 1
              ) DOWNLOAD_COUNT ON TRUE
            -- Get all the nodes for a specific meeting
            WHERE ("osf_abstractnode_tags"."tag_id" IN
                   (SELECT U0."id" AS Col1
                    FROM "osf_tag" U0
                    WHERE (U0."system" = FALSE
                           AND UPPER(U0."name" :: TEXT) = UPPER(%s)
                           AND U0."system" = FALSE))
                   AND "osf_abstractnode"."is_deleted" = FALSE
                   AND "osf_abstractnode"."is_public" = TRUE);

            """, [submission2_name, conf.name, conference_url, submission1_name, abstract_node_content_type_id, osf_user_content_type_id, conf.endpoint]
        )
        rows = cursor.fetchall()
        return [row[0] for row in rows]

def redirect_to_meetings(**kwargs):
    return redirect('/meetings/')

def serialize_conference(conf):
    return {
        'active': conf.active,
        'admins': list(conf.admins.all().values_list('guids___id', flat=True)),
        'end_date': conf.end_date,
        'endpoint': conf.endpoint,
        'field_names': conf.field_names,
        'info_url': conf.info_url,
        'is_meeting': conf.is_meeting,
        'location': conf.location,
        'logo_url': conf.logo_url,
        'name': conf.name,
        'num_submissions': conf.num_submissions,
        'poster': conf.poster,
        'public_projects': conf.public_projects,
        'start_date': conf.start_date,
        'talk': conf.talk,
    }

@ember_flag_is_active('ember_meeting_detail_page')
def conference_results(meeting):
    """Return the data for the grid view for a conference.

    :param str meeting: Endpoint name for a conference.
    """
    try:
        conf = Conference.objects.get(endpoint__iexact=meeting)
    except Conference.DoesNotExist:
        raise HTTPError(httplib.NOT_FOUND)

    data = conference_data(meeting)

    return {
        'data': data,
        'label': meeting,
        'meeting': serialize_conference(conf),
        # Needed in order to use base.mako namespace
        'settings': settings,
    }

def conference_submissions(**kwargs):
    """Return data for all OSF4M submissions.

    The total number of submissions for each meeting is calculated and cached
    in the Conference.num_submissions field.
    """
    conferences = Conference.objects.filter(is_meeting=True)
    #  TODO: Revisit this loop, there has to be a way to optimize it
    for conf in conferences:
        # For efficiency, we filter by tag first, then node
        # instead of doing a single Node query
        tags = Tag.objects.filter(system=False, name__iexact=conf.endpoint).values_list('pk', flat=True)
        nodes = AbstractNode.objects.filter(tags__in=tags, is_public=True, is_deleted=False)
        # Cache the number of submissions
        conf.num_submissions = nodes.count()
    bulk_update(conferences, update_fields=['num_submissions'])
    return {'success': True}

@ember_flag_is_active('ember_meetings_page')
def conference_view(**kwargs):
    meetings = []
    for conf in Conference.objects.all():
        if conf.num_submissions < settings.CONFERENCE_MIN_COUNT:
            continue
        if (hasattr(conf, 'is_meeting') and (conf.is_meeting is False)):
            continue
        meetings.append({
            'name': conf.name,
            'location': conf.location,
            'end_date': conf.end_date.strftime('%b %d, %Y') if conf.end_date else None,
            'start_date': conf.start_date.strftime('%b %d, %Y') if conf.start_date else None,
            'url': web_url_for('conference_results', meeting=conf.endpoint),
            'count': conf.num_submissions,
        })

    meetings.sort(key=lambda meeting: meeting['count'], reverse=True)
    return {'meetings': meetings}
<commit_msg>Restrict submissions to those that have contributors - OSF-8864, bad data.<commit_after># -*- coding: utf-8 -*-

import httplib
import logging

from django.db import transaction, connection
from django_bulk_update.helper import bulk_update
from django.contrib.contenttypes.models import ContentType

from addons.osfstorage.models import OsfStorageFile
from framework.auth import get_or_create_user
from framework.exceptions import HTTPError
from framework.flask import redirect
from framework.transactions.handlers import no_auto_transaction
from osf.models import AbstractNode, Node, Conference, Tag, OSFUser
from website import settings
from website.conferences import utils, signals
from website.conferences.message import ConferenceMessage, ConferenceError
from website.ember_osf_web.decorators import ember_flag_is_active
from website.mails import CONFERENCE_SUBMITTED, CONFERENCE_INACTIVE, CONFERENCE_FAILED
from website.mails import send_mail
from website.util import web_url_for

logger = logging.getLogger(__name__)

@no_auto_transaction
def meeting_hook():
    """View function for email conference submission.
    """
    message = ConferenceMessage()

    try:
        message.verify()
    except ConferenceError as error:
        logger.error(error)
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    try:
        conference = Conference.get_by_endpoint(message.conference_name, active=False)
    except ConferenceError as error:
        logger.error(error)
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    if not conference.active:
        send_mail(
            message.sender_email,
            CONFERENCE_INACTIVE,
            fullname=message.sender_display,
            presentations_url=web_url_for('conference_view', _absolute=True),
        )
        raise HTTPError(httplib.NOT_ACCEPTABLE)

    add_poster_by_email(conference=conference, message=message)

def add_poster_by_email(conference, message):
    """
    :param Conference conference:
    :param ConferenceMessage message:
    """
    # Fail if no attachments
    if not message.attachments:
        return send_mail(
            message.sender_email,
            CONFERENCE_FAILED,
            fullname=message.sender_display,
        )

    nodes_created = []
    users_created = []

    with transaction.atomic():
        user, user_created = get_or_create_user(
            message.sender_display,
            message.sender_email,
            is_spam=message.is_spam,
        )
        if user_created:
            user.save()  # need to save in order to access m2m fields (e.g. tags)
            users_created.append(user)
            user.add_system_tag('osf4m')
            user.update_date_last_login()
            user.save()

            # must save the user first before accessing user._id
            set_password_url = web_url_for(
                'reset_password_get',
                uid=user._id,
                token=user.verification_key_v2['token'],
                _absolute=True,
            )
        else:
            set_password_url = None

        node, node_created = Node.objects.get_or_create(
            title__iexact=message.subject,
            is_deleted=False,
            _contributors__guids___id=user._id,
            defaults={
                'title': message.subject,
                'creator': user
            }
        )
        if node_created:
            nodes_created.append(node)
            node.add_system_tag('osf4m')
            node.save()

        utils.provision_node(conference, message, node, user)
        utils.record_message(message, nodes_created, users_created)
    # Prevent circular import error
    from framework.auth import signals as auth_signals
    if user_created:
        auth_signals.user_confirmed.send(user)

    utils.upload_attachments(user, node, message.attachments)

    download_url = node.web_url_for(
        'addon_view_or_download_file',
        path=message.attachments[0].filename,
        provider='osfstorage',
        action='download',
        _absolute=True,
    )

    # Send confirmation email
    send_mail(
        message.sender_email,
        CONFERENCE_SUBMITTED,
        conf_full_name=conference.name,
        conf_view_url=web_url_for(
            'conference_results',
            meeting=message.conference_name,
            _absolute=True,
        ),
        fullname=message.sender_display,
        user_created=user_created,
        set_password_url=set_password_url,
        profile_url=user.absolute_url,
        node_url=node.absolute_url,
        file_url=download_url,
        presentation_type=message.conference_category.lower(),
        is_spam=message.is_spam,
    )
    if node_created and user_created:
        signals.osf4m_user_created.send(user, conference=conference, node=node)

## TODO DELETE
def _render_conference_node(node, idx, conf):
    record = OsfStorageFile.objects.filter(node=node).first()

    if not record:
        download_url = ''
        download_count = 0
    else:
        download_count = record.get_download_count()
        download_url = node.web_url_for(
            'addon_view_or_download_file',
            path=record.path.strip('/'),
            provider='osfstorage',
            action='download',
            _absolute=True,
        )

    author = node.visible_contributors[0]
    tags = list(node.tags.filter(system=False).values_list('name', flat=True))

    return {
        'id': idx,
        'title': node.title,
        'nodeUrl': node.url,
        'author': author.family_name if author.family_name else author.fullname,
        'authorUrl': author.url,
        'category': conf.field_names['submission1'] if conf.field_names['submission1'] in tags else conf.field_names['submission2'],
        'download': download_count,
        'downloadUrl': download_url,
        'dateCreated': node.created.isoformat(),
        'confName': conf.name,
        'confUrl': web_url_for('conference_results', meeting=conf.endpoint),
        'tags': ' '.join(tags)
    }

def conference_data(meeting):
    try:
        conf = Conference.objects.get(endpoint__iexact=meeting)
    except Conference.DoesNotExist:
        raise HTTPError(httplib.NOT_FOUND)

    return conference_submissions_sql(conf)

def conference_submissions_sql(conf):
    submission1_name = conf.field_names['submission1']
    submission2_name = conf.field_names['submission2']
    conference_url = web_url_for('conference_results', meeting=conf.endpoint)
    abstract_node_content_type_id = ContentType.objects.get_for_model(AbstractNode).id
    osf_user_content_type_id = ContentType.objects.get_for_model(OSFUser).id

    with connection.cursor() as cursor:
        cursor.execute(
            """
            SELECT  json_build_object(
                    'id', osf_abstractnode.id,
                    'title', osf_abstractnode.title,
                    'nodeUrl', '/' || guid._id || '/',
                    'author', CASE WHEN AUTHOR.family_name != '' THEN AUTHOR.family_name ELSE AUTHOR.fullname END,
                    'authorUrl', '/' || author_guid._id || '/',
                    'category', COALESCE(meeting_category.name, %s),
                    'download', COALESCE(DOWNLOAD_COUNT, 0),
                    'downloadUrl', COALESCE('/project/' || guid._id || '/files/osfstorage/' || FILE._id || '/?action=download', ''),
                    'dateCreated', osf_abstractnode.created,
                    'confName', %s,
                    'confUrl', %s,
                    'tags', tags_list.tag_list
                )
            FROM osf_abstractnode
              INNER JOIN "osf_abstractnode_tags" ON ("osf_abstractnode"."id" = "osf_abstractnode_tags"."abstractnode_id")
              LEFT JOIN (
                SELECT osf_tag.name as name, osf_abstractnode_tags.abstractnode_id as node_id
                FROM osf_tag
                INNER JOIN osf_abstractnode_tags ON (osf_tag.id = osf_abstractnode_tags.tag_id)
                WHERE (osf_tag.system = false
                       AND osf_tag.name = %s)
              ) AS meeting_category ON (osf_abstractnode.id = meeting_category.node_id)
              LEFT JOIN (
                SELECT osf_abstractnode_tags.abstractnode_id, string_agg(osf_tag.name, ' ') AS tag_list
                FROM osf_tag
                RIGHT JOIN osf_abstractnode_tags ON (osf_tag.id = osf_abstractnode_tags.tag_id)
                GROUP BY osf_abstractnode_tags.abstractnode_id
              ) AS tags_list ON (osf_abstractnode.id = tags_list.abstractnode_id)
              LEFT JOIN LATERAL (
                        SELECT osf_osfuser.*
                        FROM osf_osfuser
                          INNER JOIN osf_contributor ON (osf_contributor.user_id = osf_osfuser.id)
                        WHERE (osf_contributor.node_id = osf_abstractnode.id AND osf_contributor.visible = TRUE)
                        LIMIT 1
                        ) AUTHOR ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_guid._id
                FROM osf_guid
                WHERE (osf_guid.object_id = osf_abstractnode.id AND osf_guid.content_type_id = %s) -- Content type for AbstractNode
                ORDER BY osf_guid.created DESC
                LIMIT 1
              ) GUID ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_guid._id
                FROM osf_guid
                WHERE (osf_guid.object_id = AUTHOR.id AND osf_guid.content_type_id = %s)  -- Content type for OSFUser
                LIMIT 1
              ) AUTHOR_GUID ON TRUE
              LEFT JOIN LATERAL (
                SELECT osf_basefilenode.*
                FROM osf_basefilenode
                WHERE (
                  osf_basefilenode.type = 'osf.osfstoragefile'
                  AND osf_basefilenode.provider = 'osfstorage'
                  AND osf_basefilenode.node_id = osf_abstractnode.id
                )
                LIMIT 1
              ) FILE ON TRUE
              LEFT JOIN LATERAL (
                SELECT P.total AS DOWNLOAD_COUNT
                FROM osf_pagecounter AS P
                WHERE P._id = 'download:' || GUID._id || ':' || FILE._id
                LIMIT 1
              ) DOWNLOAD_COUNT ON TRUE
            -- Get all the nodes for a specific meeting
            WHERE ("osf_abstractnode_tags"."tag_id" IN
                   (SELECT U0."id" AS Col1
                    FROM "osf_tag" U0
                    WHERE (U0."system" = FALSE
                           AND UPPER(U0."name" :: TEXT) = UPPER(%s)
                           AND U0."system" = FALSE))
                   AND "osf_abstractnode"."is_deleted" = FALSE
                   AND "osf_abstractnode"."is_public" = TRUE
                   AND (SELECT (1) as "contributor_exists"
                        FROM osf_osfuser
                        INNER JOIN osf_contributor on osf_osfuser.id = osf_contributor.user_id
                        WHERE osf_contributor.node_id = osf_abstractnode.id
                        LIMIT 1) = 1);

            """, [submission2_name, conf.name, conference_url, submission1_name, abstract_node_content_type_id, osf_user_content_type_id, conf.endpoint]
        )
        rows = cursor.fetchall()
        return [row[0] for row in rows]

def redirect_to_meetings(**kwargs):
    return redirect('/meetings/')

def serialize_conference(conf):
    return {
        'active': conf.active,
        'admins': list(conf.admins.all().values_list('guids___id', flat=True)),
        'end_date': conf.end_date,
        'endpoint': conf.endpoint,
        'field_names': conf.field_names,
        'info_url': conf.info_url,
        'is_meeting': conf.is_meeting,
        'location': conf.location,
        'logo_url': conf.logo_url,
        'name': conf.name,
        'num_submissions': conf.num_submissions,
        'poster': conf.poster,
        'public_projects': conf.public_projects,
        'start_date': conf.start_date,
        'talk': conf.talk,
    }

@ember_flag_is_active('ember_meeting_detail_page')
def conference_results(meeting):
    """Return the data for the grid view for a conference.

    :param str meeting: Endpoint name for a conference.
    """
    try:
        conf = Conference.objects.get(endpoint__iexact=meeting)
    except Conference.DoesNotExist:
        raise HTTPError(httplib.NOT_FOUND)

    data = conference_data(meeting)

    return {
        'data': data,
        'label': meeting,
        'meeting': serialize_conference(conf),
        # Needed in order to use base.mako namespace
        'settings': settings,
    }

def conference_submissions(**kwargs):
    """Return data for all OSF4M submissions.

    The total number of submissions for each meeting is calculated and cached
    in the Conference.num_submissions field.
    """
    conferences = Conference.objects.filter(is_meeting=True)
    #  TODO: Revisit this loop, there has to be a way to optimize it
    for conf in conferences:
        # For efficiency, we filter by tag first, then node
        # instead of doing a single Node query
        tags = Tag.objects.filter(system=False, name__iexact=conf.endpoint).values_list('pk', flat=True)
        nodes = AbstractNode.objects.filter(tags__in=tags, is_public=True, is_deleted=False)
        # Cache the number of submissions
        conf.num_submissions = nodes.count()
    bulk_update(conferences, update_fields=['num_submissions'])
    return {'success': True}

@ember_flag_is_active('ember_meetings_page')
def conference_view(**kwargs):
    meetings = []
    for conf in Conference.objects.all():
        if conf.num_submissions < settings.CONFERENCE_MIN_COUNT:
            continue
        if (hasattr(conf, 'is_meeting') and (conf.is_meeting is False)):
            continue
        meetings.append({
            'name': conf.name,
            'location': conf.location,
            'end_date': conf.end_date.strftime('%b %d, %Y') if conf.end_date else None,
            'start_date': conf.start_date.strftime('%b %d, %Y') if conf.start_date else None,
            'url': web_url_for('conference_results', meeting=conf.endpoint),
            'count': conf.num_submissions,
        })

    meetings.sort(key=lambda meeting: meeting['count'], reverse=True)
    return {'meetings': meetings}
<|endoftext|>


<commit_before>#
#   Copyright 2015 Kalvin Eng
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
import database
import verify
import datetime
import menu

def create_views(cursor):
    f = open('sql/drop_views.sql')
    sql_commands = f.read()
    sql_commands = sql_commands.split(';')
    view_names = [x.strip() for x in sql_commands]
    view_names = [x.replace("drop view ","").upper() for x in view_names].pop()
    print(view_names)

    count = 0

    cursor.execute("select view_name from user_views")
    output_rows = cursor.fetchall()
    output_rows = [''.join(i) for i in output_rows]

    for line in output_rows:
        if line in view_names:
            count +=1

    if count == len(view_names):
        for x in range(0, len(sql_commands)-1):
            cursor.execute(sql_commands[x])

    query = "create view available_flights(flightno,dep_date, src,dst,dep_time,arr_time,fare,seats,\
            price, a1_name, a2_name, a1_city, a2_city) as select f.flightno, sf.dep_date, f.src, f.dst,\
            f.dep_time+(trunc(sf.dep_date)-trunc(f.dep_time)), \
            f.dep_time+(trunc(sf.dep_date)-trunc(f.dep_time))+(f.est_dur/60+a2.tzone-a1.tzone)/24, \
            fa.fare, fa.limit-count(tno), fa.price, a1.name, a2.name, a1.city, a2.city\
            from flights f, flight_fares fa, sch_flights sf, bookings b, airports a1, airports a2\
            where f.flightno=sf.flightno and f.flightno=fa.flightno and f.src=a1.acode and\
            f.dst=a2.acode and fa.flightno=b.flightno(+) and fa.fare=b.fare(+) and\
            sf.dep_date=b.dep_date(+)\
            group by f.flightno, sf.dep_date, f.src, f.dst, f.dep_time, f.est_dur,a2.tzone,\
            a1.tzone, fa.fare, fa.limit, fa.price, a1.name, a2.name, a1.city, a2.city\
            having fa.limit-count(tno) > 0"

    cursor.execute(query)

    query = "create view good_connections (src,dst,dep_date,flightno1,flightno2, layover,price, seats,\
            a1_fare, a2_fare, dep_time, arr_time, a1_name, a2_name, a1_city, a2_city) as\
            select a1.src, a2.dst, a1.dep_date, a1.flightno, a2.flightno, (a2.dep_time-a1.arr_time)*24,\
            (a1.price+a2.price),case when a1.seats <= a2.seats then a1.seats else a2.seats end,\
            a1.fare, a2.fare, a1.dep_time, a2.arr_time, a1.a1_name, a2.a2_name, a1.a1_city, a2.a2_city\
            from available_flights a1, available_flights a2 where a1.dst=a2.src and a1.src!=a2.dst and  \
            a1.arr_time<=a2.dep_time group by a1.src, a2.dst, a1.dep_date, a1.flightno, a2.flightno, a2.dep_time, \
            a1.arr_time, (a1.price+a2.price), case when a1.seats <= a2.seats then a1.seats else a2.seats end, \
            a1.fare, a2.fare, a1.dep_time, a2.arr_time, a1.a1_name, a2.a2_name, a1.a1_city, a2.a2_city"

    cursor.execute(query)

def search_flights(connection):
    cursor = database.cursor(connection)
    create_views(cursor)
    connection.commit()

    # specifying a case insensitive sort
    cursor.execute("ALTER SESSION SET NLS_COMP=LINGUISTIC")
    cursor.execute("ALTER SESSION SET NLS_SORT=BINARY_CI")

    source = ""
    destination = ""
    departure_date = ""
    sort=""
    while not(verify.char30(source)):
        source = input("Please enter a source: ").strip()
        if not(verify.char30(source)):
            print("Invalid Source Length, Try Again")

    while not(verify.char30(destination)):
        destination = input("Please enter a destination: ").strip()
        if not(verify.char30(destination)):
            print("Invalid Destination Length, Try Again")

    while not(verify.isDateFormat(departure_date)):
        departure_date= input("Please enter a departure date (DD-MON-YYYY): ").strip()
        if not(verify.isDateFormat(departure_date)):
            print("Invalid Date Format, Try Again")

    while sort!="0" and sort!="1":
        sort = input("Enter 0 to sort by price or 1 to sort by number of connections: ")

    get_acodes_query = "select acode from airports"

    valid_acodes = database.read(get_acodes_query,cursor)

    if source.upper() and destination.upper() in valid_acodes:
        search_query_gc = "select * from good_connections gc where gc.src='{}' and gc.dst='{}' and \
                        gc.dep_date = to_date('{}', 'dd-mm-yy')\
                        order by price asc, layover asc".format(source, destination, departure_date)
        search_query_af = "select * from available_flights af where af.src='{}' and af.dst='{}' and \
                        af.dep_date = to_date('{}', 'dd-mm-yy') order by price asc".format(source, destination,
                                                                                departure_date)
    else:
        search_query_gc = "select * from good_connections gc where gc.a1_name like '%{}%' or gc.a1_city like '%{}%'\
                        or gc.src  like '%{}%' and gc.a2_name like '%{}%' or gc.a1_city like '%{}%' or\
                        gc.src like '%{}%' and gc.dep_date = to_date('{}', 'dd-mon-yy') order by price asc,\
                        layover asc".format(source,source,source,destination,destination,destination,departure_date)
        search_query_af = "select * from available_flights af where af.a1_name like '%{}%' or af.a1_city like '%{}%'\
                        or af.src  like '%{}%' and af.a2_name like '%{}%' or af.a2_city like '%{}%' or\
                        af.dst like '%{}%' and af.dep_date = to_date('{}', 'dd-mon-yy')\
                        order by price asc".format(source,source,source,destination,destination,
                                            destination,departure_date)

    cursor.execute(search_query_gc)
    search_query_gc_rows = cursor.fetchall()
    cursor.execute(search_query_af)
    search_query_af_rows = cursor.fetchall()

    count = 1

    menu.clearScreen()

    print("Flights Found\n\n" + \
            "Select Row Number to book or press enter to go back\n\n")

    print(str("Row").ljust(6) + str("Fl no1").ljust(9) + str("Fl no2").ljust(9) + str("Src").ljust(5) + str("Dst").ljust(5) + str("Dep Time").ljust(10)\
        + str("Arr Time").ljust(10) + str("Stops").ljust(7) + str("Layover (hrs)").ljust(14) + str("Price").ljust(8)\
        + str("Seats").ljust(7))

    x = "-" * 90
    print(x)

    if sort == "0":
        price_rows = list()
        for row in search_query_af_rows:
            price_rows.append([row[0],"N/A",row[2],row[3],row[4].strftime('%H:%M'),row[5].strftime('%H:%M'),"0","Direct",int(row[8]),row[7]])

        for row in search_query_gc_rows:
            price_rows.append([row[3],row[4],row[0],row[1],row[10].strftime('%H:%M'),row[11].strftime('%H:%M'),"1",row[5],row[6],row[7]])

        price_rows.sort(key=lambda x:x[8])

        for row in price_rows:
            print(str(count).ljust(6) + str(row[0]).ljust(9) + str(row[1]).ljust(9) + str(row[2]).ljust(5) + str(row[3]).ljust(5) + str(row[4]).ljust(10)\
                + str(row[5]).ljust(10) + str(row[6]).ljust(7) + str(row[7]).ljust(14) + str(row[8]).ljust(8)\
                + str(row[9]).ljust(7))

    elif sort == "1":
        for row in search_query_af_rows:
            print(str(count).ljust(6) + str(row[0]).ljust(9) + str("N/A").ljust(9) + str(row[2]).ljust(5) + str(row[3]).ljust(5) + str(row[4].strftime('%H:%M')).ljust(10)\
                + str(row[5].strftime('%H:%M')).ljust(10) + "0".ljust(7) + str("Direct").ljust(14) + str(int(row[8])).ljust(8)\
                + str(row[7]).ljust(7))
            count+=1

        for row in search_query_gc_rows:
            print(str(count).ljust(6) + str(row[3]).ljust(9) + str(row[4]).ljust(9) + str(row[0]).ljust(5) + str(row[1]).ljust(5) + str(row[10].strftime('%H:%M')).ljust(10)\
                + str(row[11].strftime('%H:%M')).ljust(10) + "1".ljust(7) + str(row[5]).ljust(14) + str(row[6]).ljust(8)\
                + str(row[7]).ljust(7))
            count += 1

    while True:
        entry = input("\n")
        if entry == "":
            break<commit_msg>Fixed if no flights are found searching<commit_after>#
#   Copyright 2015 Kalvin Eng
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
import database
import verify
import datetime
import menu

def create_views(cursor):
    f = open('sql/drop_views.sql')
    sql_commands = f.read()
    sql_commands = sql_commands.split(';')
    view_names = [x.strip() for x in sql_commands]
    view_names = [x.replace("drop view ","").upper() for x in view_names].pop()
    print(view_names)

    count = 0

    cursor.execute("select view_name from user_views")
    output_rows = cursor.fetchall()
    output_rows = [''.join(i) for i in output_rows]

    for line in output_rows:
        if line in view_names:
            count +=1

    if count == len(view_names):
        for x in range(0, len(sql_commands)-1):
            cursor.execute(sql_commands[x])

    query = "create view available_flights(flightno,dep_date, src,dst,dep_time,arr_time,fare,seats,\
            price, a1_name, a2_name, a1_city, a2_city) as select f.flightno, sf.dep_date, f.src, f.dst,\
            f.dep_time+(trunc(sf.dep_date)-trunc(f.dep_time)), \
            f.dep_time+(trunc(sf.dep_date)-trunc(f.dep_time))+(f.est_dur/60+a2.tzone-a1.tzone)/24, \
            fa.fare, fa.limit-count(tno), fa.price, a1.name, a2.name, a1.city, a2.city\
            from flights f, flight_fares fa, sch_flights sf, bookings b, airports a1, airports a2\
            where f.flightno=sf.flightno and f.flightno=fa.flightno and f.src=a1.acode and\
            f.dst=a2.acode and fa.flightno=b.flightno(+) and fa.fare=b.fare(+) and\
            sf.dep_date=b.dep_date(+)\
            group by f.flightno, sf.dep_date, f.src, f.dst, f.dep_time, f.est_dur,a2.tzone,\
            a1.tzone, fa.fare, fa.limit, fa.price, a1.name, a2.name, a1.city, a2.city\
            having fa.limit-count(tno) > 0"

    cursor.execute(query)

    query = "create view good_connections (src,dst,dep_date,flightno1,flightno2, layover,price, seats,\
            a1_fare, a2_fare, dep_time, arr_time, a1_name, a2_name, a1_city, a2_city) as\
            select a1.src, a2.dst, a1.dep_date, a1.flightno, a2.flightno, (a2.dep_time-a1.arr_time)*24,\
            (a1.price+a2.price),case when a1.seats <= a2.seats then a1.seats else a2.seats end,\
            a1.fare, a2.fare, a1.dep_time, a2.arr_time, a1.a1_name, a2.a2_name, a1.a1_city, a2.a2_city\
            from available_flights a1, available_flights a2 where a1.dst=a2.src and a1.src!=a2.dst and  \
            a1.arr_time<=a2.dep_time group by a1.src, a2.dst, a1.dep_date, a1.flightno, a2.flightno, a2.dep_time, \
            a1.arr_time, (a1.price+a2.price), case when a1.seats <= a2.seats then a1.seats else a2.seats end, \
            a1.fare, a2.fare, a1.dep_time, a2.arr_time, a1.a1_name, a2.a2_name, a1.a1_city, a2.a2_city"

    cursor.execute(query)

def search_flights(connection):
    cursor = database.cursor(connection)
    create_views(cursor)
    connection.commit()

    # specifying a case insensitive sort
    cursor.execute("ALTER SESSION SET NLS_COMP=LINGUISTIC")
    cursor.execute("ALTER SESSION SET NLS_SORT=BINARY_CI")

    source = ""
    destination = ""
    departure_date = ""
    sort=""
    while not(verify.char30(source)):
        source = input("Please enter a source: ").strip()
        if not(verify.char30(source)):
            print("Invalid Source Length, Try Again")

    while not(verify.char30(destination)):
        destination = input("Please enter a destination: ").strip()
        if not(verify.char30(destination)):
            print("Invalid Destination Length, Try Again")

    while not(verify.isDateFormat(departure_date)):
        departure_date= input("Please enter a departure date (DD-MON-YYYY): ").strip()
        if not(verify.isDateFormat(departure_date)):
            print("Invalid Date Format, Try Again")

    while sort!="0" and sort!="1":
        sort = input("Enter 0 to sort by price or 1 to sort by number of connections: ")

    get_acodes_query = "select acode from airports"

    valid_acodes = database.read(get_acodes_query,cursor)

    if source.upper() and destination.upper() in valid_acodes:
        search_query_gc = "select * from good_connections gc where gc.src='{}' and gc.dst='{}' and \
                        gc.dep_date = to_date('{}', 'dd-mm-yy')\
                        order by price asc, layover asc".format(source, destination, departure_date)
        search_query_af = "select * from available_flights af where af.src='{}' and af.dst='{}' and \
                        af.dep_date = to_date('{}', 'dd-mm-yy') order by price asc".format(source, destination,
                                                                                departure_date)
    else:
        search_query_gc = "select * from good_connections gc where gc.a1_name like '%{}%' or gc.a1_city like '%{}%'\
                        or gc.src  like '%{}%' and gc.a2_name like '%{}%' or gc.a1_city like '%{}%' or\
                        gc.src like '%{}%' and gc.dep_date = to_date('{}', 'dd-mon-yy') order by price asc,\
                        layover asc".format(source,source,source,destination,destination,destination,departure_date)
        search_query_af = "select * from available_flights af where af.a1_name like '%{}%' or af.a1_city like '%{}%'\
                        or af.src  like '%{}%' and af.a2_name like '%{}%' or af.a2_city like '%{}%' or\
                        af.dst like '%{}%' and af.dep_date = to_date('{}', 'dd-mon-yy')\
                        order by price asc".format(source,source,source,destination,destination,
                                            destination,departure_date)

    cursor.execute(search_query_gc)
    search_query_gc_rows = cursor.fetchall()
    cursor.execute(search_query_af)
    search_query_af_rows = cursor.fetchall()

    count = 1

    menu.clearScreen()

    print("Flights Found\n\n" + \
            "Select Row Number to book or press enter to go back\n\n")

    print(str("Row").ljust(6) + str("Fl no1").ljust(9) + str("Fl no2").ljust(9) + str("Src").ljust(5) + str("Dst").ljust(5) + str("Dep Time").ljust(10)\
        + str("Arr Time").ljust(10) + str("Stops").ljust(7) + str("Layover (hrs)").ljust(14) + str("Price").ljust(8)\
        + str("Seats").ljust(7))

    x = "-" * 90
    print(x)

    if len(search_query_af_rows) == 0 and len(search_query_gc_rows) == 0:
        print("No flights found")

    elif sort == "0":
        price_rows = list()
        for row in search_query_af_rows:
            price_rows.append([row[0],"N/A",row[2],row[3],row[4].strftime('%H:%M'),row[5].strftime('%H:%M'),"0","Direct",int(row[8]),row[7]])

        for row in search_query_gc_rows:
            price_rows.append([row[3],row[4],row[0],row[1],row[10].strftime('%H:%M'),row[11].strftime('%H:%M'),"1",row[5],row[6],row[7]])

        price_rows.sort(key=lambda x:x[8])

        for row in price_rows:
            print(str(count).ljust(6) + str(row[0]).ljust(9) + str(row[1]).ljust(9) + str(row[2]).ljust(5) + str(row[3]).ljust(5) + str(row[4]).ljust(10)\
                + str(row[5]).ljust(10) + str(row[6]).ljust(7) + str(row[7]).ljust(14) + str(row[8]).ljust(8)\
                + str(row[9]).ljust(7))

    elif sort == "1":
        for row in search_query_af_rows:
            print(str(count).ljust(6) + str(row[0]).ljust(9) + str("N/A").ljust(9) + str(row[2]).ljust(5) + str(row[3]).ljust(5) + str(row[4].strftime('%H:%M')).ljust(10)\
                + str(row[5].strftime('%H:%M')).ljust(10) + "0".ljust(7) + str("Direct").ljust(14) + str(int(row[8])).ljust(8)\
                + str(row[7]).ljust(7))
            count+=1

        for row in search_query_gc_rows:
            print(str(count).ljust(6) + str(row[3]).ljust(9) + str(row[4]).ljust(9) + str(row[0]).ljust(5) + str(row[1]).ljust(5) + str(row[10].strftime('%H:%M')).ljust(10)\
                + str(row[11].strftime('%H:%M')).ljust(10) + "1".ljust(7) + str(row[5]).ljust(14) + str(row[6]).ljust(8)\
                + str(row[7]).ljust(7))
            count += 1

    while True:
        entry = input("\n")
        if entry == "":
            break<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# seeder.py - Exports reachable nodes into a DNS zone file for DNS seeder.
#
# Copyright (c) 2014 Addy Yeow Chin Heng <ayeowch@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Exports reachable nodes into a DNS zone file for DNS seeder.
"""

import glob
import json
import logging
import operator
import os
import random
import sys
import threading
import time
from ConfigParser import ConfigParser

from protocol import DEFAULT_PORT

SETTINGS = {}

def export_nodes(nodes):
    """
    Exports nodes as A and AAAA records into DNS zone file. Nodes are selected
    from oldest (longest uptime) to newest each with unique AS number.
    """
    nodes = sorted(nodes, key=operator.itemgetter(4))[:SETTINGS['nodes']]
    min_height = SETTINGS['min_height']
    min_age = SETTINGS['min_age']
    now = int(time.time())
    auth_nodes = [node for node in nodes if node[0] == SETTINGS['auth_node']]
    if len(auth_nodes) > 0:
        min_height = auth_nodes[0][5]
    logging.info("Min. height: {}".format(min_height))
    asns = []
    a_records = []
    aaaa_records = []
    for node in nodes:
        address = node[0]
        port = node[1]
        age = now - node[4]
        height = node[5]
        asn = node[12]
        if (port == DEFAULT_PORT and asn not in asns and
                age >= min_age and height >= min_height):
            if ":" in address:
                aaaa_records.append("@\tIN\tAAAA\t{}".format(address))
            else:
                a_records.append("@\tIN\tA\t{}".format(address))
            asns.append(asn)
    random.shuffle(a_records)
    random.shuffle(aaaa_records)
    logging.info("A records: {}".format(len(a_records)))
    logging.info("AAAA records: {}".format(len(aaaa_records)))
    a_records = "\n".join(a_records[:SETTINGS['a_records']]) + "\n"
    aaaa_records = "\n".join(aaaa_records[:SETTINGS['aaaa_records']]) + "\n"
    template = open(SETTINGS['template'], "r").read()
    open(SETTINGS['zone_file'], "w").write(
        template + a_records + aaaa_records)

def cron():
    """
    Periodically fetches latest snapshot to sample nodes for DNS zone file.
    """
    while True:
        time.sleep(5)
        dump = max(glob.iglob("{}/*.json".format(SETTINGS['export_dir'])))
        logging.info("Dump: {}".format(dump))
        try:
            nodes = json.loads(open(dump, "r").read(), encoding="latin-1")
        except ValueError:
            logging.warning("Write pending")
        else:
            export_nodes(nodes)

def init_settings(argv):
    """
    Populates SETTINGS with key-value pairs from configuration file.
    """
    conf = ConfigParser()
    conf.read(argv[1])
    SETTINGS['logfile'] = conf.get('seeder', 'logfile')
    SETTINGS['debug'] = conf.getboolean('seeder', 'debug')
    SETTINGS['export_dir'] = conf.get('seeder', 'export_dir')
    SETTINGS['nodes'] = conf.getint('seeder', 'nodes')
    SETTINGS['min_height'] = conf.getint('seeder', 'min_height')
    SETTINGS['min_age'] = conf.getint('seeder', 'min_age')
    SETTINGS['auth_node'] = conf.get('seeder', 'auth_node')
    SETTINGS['zone_file'] = conf.get('seeder', 'zone_file')
    SETTINGS['template'] = conf.get('seeder', 'template')
    SETTINGS['a_records'] = conf.getint('seeder', 'a_records')
    SETTINGS['aaaa_records'] = conf.getint('seeder', 'aaaa_records')

def main(argv):
    if len(argv) < 2 or not os.path.exists(argv[1]):
        print("Usage: seeder.py [config]")
        return 1

    # Initialize global settings
    init_settings(argv)

    # Initialize logger
    loglevel = logging.INFO
    if SETTINGS['debug']:
        loglevel = logging.DEBUG

    logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
                 "%(message)s")
    logging.basicConfig(level=loglevel,
                        format=logformat,
                        filename=SETTINGS['logfile'],
                        filemode='w')
    print("Writing output to {}, press CTRL+C to terminate..".format(
          SETTINGS['logfile']))

    threading.Thread(target=cron).start()

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>Align min. age with oldest node if necessary<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# seeder.py - Exports reachable nodes into a DNS zone file for DNS seeder.
#
# Copyright (c) 2014 Addy Yeow Chin Heng <ayeowch@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Exports reachable nodes into a DNS zone file for DNS seeder.
"""

import glob
import json
import logging
import operator
import os
import random
import sys
import threading
import time
from ConfigParser import ConfigParser

from protocol import DEFAULT_PORT

SETTINGS = {}

def export_nodes(nodes):
    """
    Exports nodes as A and AAAA records into DNS zone file. Nodes are selected
    from oldest (longest uptime) to newest each with unique AS number.
    """
    nodes = sorted(nodes, key=operator.itemgetter(4))[:SETTINGS['nodes']]
    min_height = SETTINGS['min_height']
    min_age = SETTINGS['min_age']
    now = int(time.time())
    auth_nodes = [node for node in nodes if node[0] == SETTINGS['auth_node']]
    if len(auth_nodes) > 0:
        min_height = auth_nodes[0][5]
    logging.info("Min. height: {}".format(min_height))
    oldest = now - min(nodes, key=operator.itemgetter(4))[4]
    if oldest < min_age:
        min_age = oldest
    logging.info("Min. age: {}".format(min_age))
    asns = []
    a_records = []
    aaaa_records = []
    for node in nodes:
        address = node[0]
        port = node[1]
        age = now - node[4]
        height = node[5]
        asn = node[12]
        if (port == DEFAULT_PORT and asn not in asns and
                age >= min_age and height >= min_height):
            if ":" in address:
                aaaa_records.append("@\tIN\tAAAA\t{}".format(address))
            else:
                a_records.append("@\tIN\tA\t{}".format(address))
            asns.append(asn)
    random.shuffle(a_records)
    random.shuffle(aaaa_records)
    logging.info("A records: {}".format(len(a_records)))
    logging.info("AAAA records: {}".format(len(aaaa_records)))
    a_records = "\n".join(a_records[:SETTINGS['a_records']]) + "\n"
    aaaa_records = "\n".join(aaaa_records[:SETTINGS['aaaa_records']]) + "\n"
    template = open(SETTINGS['template'], "r").read()
    open(SETTINGS['zone_file'], "w").write(
        template + a_records + aaaa_records)

def cron():
    """
    Periodically fetches latest snapshot to sample nodes for DNS zone file.
    """
    while True:
        time.sleep(5)
        dump = max(glob.iglob("{}/*.json".format(SETTINGS['export_dir'])))
        logging.info("Dump: {}".format(dump))
        try:
            nodes = json.loads(open(dump, "r").read(), encoding="latin-1")
        except ValueError:
            logging.warning("Write pending")
        else:
            export_nodes(nodes)

def init_settings(argv):
    """
    Populates SETTINGS with key-value pairs from configuration file.
    """
    conf = ConfigParser()
    conf.read(argv[1])
    SETTINGS['logfile'] = conf.get('seeder', 'logfile')
    SETTINGS['debug'] = conf.getboolean('seeder', 'debug')
    SETTINGS['export_dir'] = conf.get('seeder', 'export_dir')
    SETTINGS['nodes'] = conf.getint('seeder', 'nodes')
    SETTINGS['min_height'] = conf.getint('seeder', 'min_height')
    SETTINGS['min_age'] = conf.getint('seeder', 'min_age')
    SETTINGS['auth_node'] = conf.get('seeder', 'auth_node')
    SETTINGS['zone_file'] = conf.get('seeder', 'zone_file')
    SETTINGS['template'] = conf.get('seeder', 'template')
    SETTINGS['a_records'] = conf.getint('seeder', 'a_records')
    SETTINGS['aaaa_records'] = conf.getint('seeder', 'aaaa_records')

def main(argv):
    if len(argv) < 2 or not os.path.exists(argv[1]):
        print("Usage: seeder.py [config]")
        return 1

    # Initialize global settings
    init_settings(argv)

    # Initialize logger
    loglevel = logging.INFO
    if SETTINGS['debug']:
        loglevel = logging.DEBUG

    logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
                 "%(message)s")
    logging.basicConfig(level=loglevel,
                        format=logformat,
                        filename=SETTINGS['logfile'],
                        filemode='w')
    print("Writing output to {}, press CTRL+C to terminate..".format(
          SETTINGS['logfile']))

    threading.Thread(target=cron).start()

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################

from openerp import models, api

class StockQuant(models.Model):
    _inherit = 'stock.quant'

    @api.multi
    def merge_stock_quants(self):
        pending_quants_ids = self.ids
        for quant2merge in self:
            if (quant2merge.id in pending_quants_ids and
                    not quant2merge.reservation_id):
                quants = self.search(
                    [('id', '!=', quant2merge.id),
                     ('product_id', '=', quant2merge.product_id.id),
                     ('lot_id', '=', quant2merge.lot_id.id),
                     ('package_id', '=', quant2merge.package_id.id),
                     ('location_id', '=', quant2merge.location_id.id),
                     ('reservation_id', '=', False),
                     ('propagated_from_id', '=',
                      quant2merge.propagated_from_id.id)])
                for quant in quants:
                    if (self._get_latest_move(quant2merge) ==
                            self._get_latest_move(quant)):
                        quant2merge.qty += quant.qty
                        quant2merge.cost += quant.cost
                        if quant.id in pending_quants_ids:
                            pending_quants_ids.remove(quant.id)
                        quant.sudo().unlink()

    @api.model
    def quants_unreserve(self, move):
        quants = move.reserved_quant_ids
        super(StockQuant, self).quants_unreserve(move)
        quants.merge_stock_quants()
<commit_msg>[MOD] Use browse record instead of ids<commit_after># -*- encoding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################

from openerp import models, api

class StockQuant(models.Model):
    _inherit = 'stock.quant'

    @api.multi
    def merge_stock_quants(self):
        pending_quants = self.filtered(lambda x: True)
        for quant2merge in self:
            if (quant2merge in pending_quants and
                    not quant2merge.reservation_id):
                quants = self.search(
                    [('id', '!=', quant2merge.id),
                     ('product_id', '=', quant2merge.product_id.id),
                     ('lot_id', '=', quant2merge.lot_id.id),
                     ('package_id', '=', quant2merge.package_id.id),
                     ('location_id', '=', quant2merge.location_id.id),
                     ('reservation_id', '=', False),
                     ('propagated_from_id', '=',
                      quant2merge.propagated_from_id.id)])
                for quant in quants:
                    if (self._get_latest_move(quant2merge) ==
                            self._get_latest_move(quant)):
                        quant2merge.qty += quant.qty
                        quant2merge.cost += quant.cost
                        pending_quants -= quant
                        quant.sudo().unlink()

    @api.model
    def quants_unreserve(self, move):
        quants = move.reserved_quant_ids
        super(StockQuant, self).quants_unreserve(move)
        quants.merge_stock_quants()
<|endoftext|>


<commit_before>import markdown
from datetime import datetime
from twitter_text import TwitterText
from flask import Markup

def relative_time(timestamp):
    delta = (datetime.now() - datetime.fromtimestamp(timestamp))
    delta_s = delta.days * 86400 + delta.seconds
    if delta_s < 60:
        return "less than a minute ago"
    elif delta_s < 120:
        return "about a minute ago"
    elif delta_s < (60 * 60):
        return str(delta_s / 60) + " minutes ago"
    elif delta_s < (120 * 60):
        return "about an hour ago"
    elif delta_s < (24 * 60 * 60):
        return "about " + str(delta_s / 3600) + " hours ago"
    elif delta_s < (48 * 60 * 60):
        return "1 day ago"
    else:
        return str(delta_s / 86400) + " days ago"

def markup_tweet(text):
    return Markup(TwitterText(text).autolink.auto_link())

def markup_markdown(text):
    return Markup(markdown.markdown(text))
<commit_msg>Update Markdown filter to recognize metadata.<commit_after>import markdown
from datetime import datetime
from twitter_text import TwitterText
from flask import Markup

def relative_time(timestamp):
    delta = (datetime.now() - datetime.fromtimestamp(timestamp))
    delta_s = delta.days * 86400 + delta.seconds
    if delta_s < 60:
        return "less than a minute ago"
    elif delta_s < 120:
        return "about a minute ago"
    elif delta_s < (60 * 60):
        return str(delta_s / 60) + " minutes ago"
    elif delta_s < (120 * 60):
        return "about an hour ago"
    elif delta_s < (24 * 60 * 60):
        return "about " + str(delta_s / 3600) + " hours ago"
    elif delta_s < (48 * 60 * 60):
        return "1 day ago"
    else:
        return str(delta_s / 86400) + " days ago"

def markup_tweet(text):
    return Markup(TwitterText(text).autolink.auto_link())

def markup_markdown(text):
    md = markdown.Markdown(extensions=['meta'])
    return Markup(md.convert(text))
<|endoftext|>


<commit_before>import copy
import numpy as np
from Orange import data
from ..data.value import Value

class Fitter:
    """
    Return a model build from the data given as a :class:`data.Table`

    Models can be implemented using one or two classes.

    In a two-class model, the first class is derived from `Fitter` and overloads
    the `__call__` method. The method gets a :class:`~Orange.data.Table` and
    returns an instance of the corresponding Model class. The `Model` should
    implement method `predict` (see documentation on :class:`Model`).

    In a single-class model, the class should be derived from :class:`Fitter`
    and :class:`Model` and provide two methods:

    - `fit` gets X, Y, and weights (or `None`) as arguments and initializes the
       model's parameters

    - `predict` (see documentation on :class:`~Orange.classification.Model`).
    """

class Model:
    Value = 0
    Probs = 1
    ValueProbs =2

    def __init__(self, domain=None):
        if isinstance(self, Fitter):
            domain = None
        elif not domain:
            raise ValueError("unspecified domain")
        self.domain = domain

    def predict(self, data):
        raise TypeError("Descendants of Model must overload method predict")

    def __call__(self, data, ret=Value):
        # Check if this is an unfitted single-class model
        if isinstance(self, Fitter) and self.domain is None:
            model = copy.deepcopy(self)
            model.fit(data.X, data.Y, data.W if data.has_weights else None)
            model.domain = data.domain
            return model

        if not 0 <= ret <= 2:
            raise ValueError("invalid value of argument 'ret'")
        if ret > 0 and any(isinstance(v, data.ContinuousVariable)
                           for v in self.domain.class_vars):
            raise ValueError("cannot predict continuous distributions")

        # Call the predictor
        if isinstance(data, np.array):
            prediction = self.predict(np.atleast_2d(data))
        elif isinstance(data, data.Instance):
            if data.domain != self.domain:
                data = data.Instance(self.domain, data)
            prediction = self.predict(np.atleast_2d(data._values))
        elif isinstance(data, data.Table):
            if data.domain != self.domain:
                data = data.Table.new_from_table(self.domain, data)
            prediction = self.predict(data.X)
        else:
            raise TypeError("Unrecognized argument (instance of '%s')",
                            type(data).__name__)

        # Parse the result into value and probs
        multitarget = len(self.domain.class_vars) > 1
        if isinstance(prediction, tuple):
            value, probs = prediction
        elif prediction.ndim == 1 + multitarget:
            value, probs = prediction, None
        elif prediction.ndim == 2 + multitarget:
            value, probs = None, prediction
        else:
            raise TypeError("model returned a %i-dimensional array",
                            prediction.ndim)

        # Ensure that we have what we need to return
        if ret != Model.Probs and value is None:
            value = np.argmax(probs, axis=1)
        if ret != Model.Value and probs is None:
            if multitarget:
                max_card = max(len(c.values) for c in self.domain.class_vars)
                probs = np.zeros(value.shape + (max_card,), float)
                for i, cvar in enumerate(self.domain.class_vars):
                    for cval in len(cvar.values):
                        probs[i, value[:, i]==cval] = 1
            else:
                probs = np.zeros((len(value), len(self.domain.class_var.values)),
                                 float)
                for cval in probs.shape[1]:
                    probs[value==cval] = 1
            return probs

        # Return what we need to
        if ret == Model.Value:
            if isinstance(data, data.Instance) and not multitarget:
                value = Value(self.domain.class_var, value[0])
            return value
        if ret == Model.Probs:
            return probs
        else: # ret == Model.ValueProbs
            return value, probs
<commit_msg>Base classifier now uses bottlechest's bincount to simplify construction of probabilities from values<commit_after>import copy
import numpy as np
import bottleneck as bn
from Orange import data
from ..data.value import Value

class Fitter:
    """
    Return a model build from the data given as a :class:`data.Table`

    Models can be implemented using one or two classes.

    In a two-class model, the first class is derived from `Fitter` and overloads
    the `__call__` method. The method gets a :class:`~Orange.data.Table` and
    returns an instance of the corresponding Model class. The `Model` should
    implement method `predict` (see documentation on :class:`Model`).

    In a single-class model, the class should be derived from :class:`Fitter`
    and :class:`Model` and provide two methods:

    - `fit` gets X, Y, and weights (or `None`) as arguments and initializes the
       model's parameters

    - `predict` (see documentation on :class:`~Orange.classification.Model`).
    """

class Model:
    Value = 0
    Probs = 1
    ValueProbs =2

    def __init__(self, domain=None):
        if isinstance(self, Fitter):
            domain = None
        elif not domain:
            raise ValueError("unspecified domain")
        self.domain = domain

    def predict(self, data):
        raise TypeError("Descendants of Model must overload method predict")

    def __call__(self, data, ret=Value):
        # Check if this is an unfitted single-class model
        if isinstance(self, Fitter) and self.domain is None:
            model = copy.deepcopy(self)
            model.fit(data.X, data.Y, data.W if data.has_weights else None)
            model.domain = data.domain
            return model

        if not 0 <= ret <= 2:
            raise ValueError("invalid value of argument 'ret'")
        if ret > 0 and any(isinstance(v, data.ContinuousVariable)
                           for v in self.domain.class_vars):
            raise ValueError("cannot predict continuous distributions")

        # Call the predictor
        if isinstance(data, np.array):
            prediction = self.predict(np.atleast_2d(data))
        elif isinstance(data, data.Instance):
            if data.domain != self.domain:
                data = data.Instance(self.domain, data)
            prediction = self.predict(np.atleast_2d(data._values))
        elif isinstance(data, data.Table):
            if data.domain != self.domain:
                data = data.Table.new_from_table(self.domain, data)
            prediction = self.predict(data.X)
        else:
            raise TypeError("Unrecognized argument (instance of '%s')",
                            type(data).__name__)

        # Parse the result into value and probs
        multitarget = len(self.domain.class_vars) > 1
        if isinstance(prediction, tuple):
            value, probs = prediction
        elif prediction.ndim == 1 + multitarget:
            value, probs = prediction, None
        elif prediction.ndim == 2 + multitarget:
            value, probs = None, prediction
        else:
            raise TypeError("model returned a %i-dimensional array",
                            prediction.ndim)

        # Ensure that we have what we need to return
        if ret != Model.Probs and value is None:
            value = np.argmax(probs, axis=1)
        if ret != Model.Value and probs is None:
            if multitarget:
                max_card = max(len(c.values) for c in self.domain.class_vars)
                probs = np.zeros(value.shape + (max_card,), float)
                for i, cvar in enumerate(self.domain.class_vars):
                    probs[i] = bn.bincount(np.atleast_2d(value[:, i]), max_card)
            else:
                probs = bn.bincount(np.atleast_2d(value),
                                    len(self.domain.class_var.values))
            return probs

        # Return what we need to
        if ret == Model.Value:
            if isinstance(data, data.Instance) and not multitarget:
                value = Value(self.domain.class_var, value[0])
            return value
        if ret == Model.Probs:
            return probs
        else: # ret == Model.ValueProbs
            return value, probs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function

import socket
import argparse
import time
from contextlib import closing
import errno

from auxiliaries import set_verbosity, Config, PublicKey
from globalvalues import DEFAULT_HOSTNAME, DEFAULT_SENDER_MODE
from globalvalues import DEFAULT_UDP_PORT, DEFAULT_TCP_PORT
from globalvalues import DEFAULT_CONFIG, DEFAULT_PUBLICKEY

TCP_TIMEOUT = 5

class ServerSender(object):
    """
    Provides ServerSender.send_cpm() for sending UDP packets to the DoseNet
    server.
    """

    def __init__(self,
                 manager=None,
                 network_status=None,
                 address=DEFAULT_HOSTNAME,
                 port=None,
                 config=None,
                 publickey=None,
                 verbosity=1,
                 logfile=None,
                 mode=None,
                 ):
        """
        network_status, config, publickey loaded from manager if not provided.
        address and port take system defaults, although without config and
          publickey, address and port will not be used.
        """

        self.v = verbosity
        if manager and logfile is None:
            set_verbosity(self, logfile=manager.logfile)
        else:
            set_verbosity(self, logfile=logfile)

        self.handle_input(
            manager, mode, port, network_status, config, publickey)

        self.tcp_sender = None
        self.address = address

    def handle_input(
            self, manager, mode, port, network_status, config, publickey):

        # TODO: this stuff is messy. Is there a cleaner way using exceptions?
        if manager is None:
            self.vprint(1, 'ServerSender starting without Manager object')
        self.manager = manager

        try:
            if mode is None:
                self.mode = DEFAULT_SENDER_MODE
            elif mode.lower() == 'udp':
                self.mode = 'udp'
            elif mode.lower() == 'tcp':
                self.mode = 'tcp'
            else:
                raise RuntimeError(
                    'Invalid ServerSender mode (choose TCP or UDP)')
        except AttributeError:
            raise RuntimeError('Invalid ServerSender mode (choose TCP or UDP)')

        if self.mode == 'udp':
            self.port = DEFAULT_UDP_PORT
        elif self.mode == 'tcp':
            self.port = DEFAULT_TCP_PORT

        if network_status is None:
            if manager is None:
                self.vprint(
                    1, 'ServerSender starting without network status object')
                self.network_up = None
            else:
                self.network_up = manager.network_up
        else:
            self.network_up = network_status

        if config is None:
            if manager is None:
                self.vprint(1, 'ServerSender starting without config file')
                self.config = None
            else:
                self.config = manager.config
        else:
            self.config = config

        if publickey is None:
            if manager is None:
                self.vprint(1, 'ServerSender starting without publickey file')
                self.encrypter = None
            elif manager.publickey is None:
                self.vprint(1, 'ServerSender starting without publickey file')
                self.encrypter = None
            else:
                self.encrypter = manager.publickey.encrypter
        else:
            self.encrypter = publickey.encrypter

    def construct_packet(self, cpm, cpm_error, error_code=0):
        """
        Construct the raw packet string. (basically copied from old code)
        """

        c = ','
        try:
            raw_packet = (
                str(self.config.hash) + c +
                str(self.config.ID) + c +
                str(cpm) + c +
                str(cpm_error) + c +
                str(error_code))
        except AttributeError:      # on self.config.hash
            raise MissingFile('Missing or broken Config object')
        else:
            self.vprint(3, 'Constructed packet')
            return raw_packet

    def encrypt_packet(self, raw_packet):
        """Encrypt the raw packet"""

        self.vprint(3, 'Encrypting packet: {}'.format(raw_packet))
        try:
            encrypted = self.encrypter.encrypt_message(raw_packet)[0]
        except AttributeError:
            raise MissingFile('Missing or broken PublicKey object')
        else:
            return encrypted

    def send_data(self, encrypted):
        """
        Send data according to self.mode, and handle common errors
        """

        self.vprint(3, 'Trying to send data by {}'.format(self.mode))
        try:
            if self.mode == 'udp':
                self.send_udp(encrypted)
            elif self.mode == 'tcp':
                self.send_tcp(encrypted)
        except socket.gaierror as e:
            if e[0] == socket.EAI_AGAIN:
                # TCP and UDP
                # network is down, but NetworkStatus didn't notice yet
                self.vprint(
                    1, 'Failed to send packet! Address resolution error')
                self.network_up.update()
            else:
                self.vprint(1, 'Failed to send packet! Address error: ' +
                            '{}: {}'.format(*e))
        except socket.error as e:
            if e[0] == errno.ECONNREFUSED:
                # TCP
                # server is not accepting connections
                self.vprint(1, 'Failed to send packet! Connection refused')
            else:
                # consider handling errno.ECONNABORTED, errno.ECONNRESET
                self.vprint(1, 'Failed to send packet! Socket error: ' +
                            '{}: {}'.format(*e))
        except socket.timeout:
            # TCP
            self.vprint(1, 'Failed to send packet! Socket timeout')
            self.network_up.update()

    def send_udp(self, encrypted):
        """
        Send the encrypted packet over UDP
        """

        self.vprint(3, 'Sending encrypted UDP packet to {}:{}'.format(
            self.address, self.port))
        with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as s:
            s.sendto(encrypted, (self.address, self.port))
            self.vprint(
                3, 'UDP packet sent successfully (no client-side error)')

    def send_tcp(self, encrypted):
        """
        Send the encrypted packet over TCP
        """

        self.vprint(3, 'Sending encrypted TCP packet to {}:{}'.format(
            self.address, self.port))
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
            s.settimeout(TCP_TIMEOUT)   # generous timeout
            s.connect((self.address, self.port))
            s.sendall(encrypted)
            self.vprint(3, 'TCP packet sent successfully')

    def send_cpm(self, cpm, cpm_error, error_code=0):
        """Construct, encrypt, and send the packet"""

        packet = self.construct_packet(cpm, cpm_error, error_code=error_code)
        encrypted = self.encrypt_packet(packet)
        if self.network_up or self.network_up is None:
            self.send_data(encrypted)
        else:
            # TODO: feature add here
            self.vprint(2, 'Network DOWN, not sending packet')

class PacketError(Exception):
    pass

class MissingFile(PacketError):
    pass

def send_test_packets(
        mode=DEFAULT_SENDER_MODE,
        config=DEFAULT_CONFIG,
        publickey=DEFAULT_PUBLICKEY,
        address=DEFAULT_HOSTNAME,
        n=3):
    """
    Send n (default 3) test packets to the DoseNet server.
    """

    sleep_time = 2      # seconds

    try:
        config_obj = Config(config)
    except IOError:
        # file doesn't exist
        config_obj = None
    key_obj = PublicKey(publickey)

    sender = ServerSender(
        mode=mode, address=address,
        config=config_obj, publickey=key_obj, verbosity=3)

    try:
        station_id = config_obj.ID
    except AttributeError:
        station_id = '?'
    raw_packet = 'Test packet from station {} by mode {}'.format(
        station_id, mode)

    encrypted = sender.encrypt_packet(raw_packet)

    for _ in xrange(n):
        sender.send_data(encrypted)
        time.sleep(sleep_time)

if __name__ == '__main__':
    # send test packets
    parser = argparse.ArgumentParser(
        description='Sender for UDP/TCP data packets. ' +
        'Normally called from manager.py. ' +
        'Called directly, it will send 3 test packets to server. ' +
        'Specify udp or tcp on command line.')
    parser.add_argument(
        'mode', choices=['udp', 'tcp'], nargs='?', default='udp',
        help='The network protocol to use in sending test packets')
    args = parser.parse_args()

    send_test_packets(
        mode=args.mode, config=DEFAULT_CONFIG, publickey=DEFAULT_PUBLICKEY)
<commit_msg>handle Network Unreachable error for IP address<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function

import socket
import argparse
import time
from contextlib import closing
import errno

from auxiliaries import set_verbosity, Config, PublicKey
from globalvalues import DEFAULT_HOSTNAME, DEFAULT_SENDER_MODE
from globalvalues import DEFAULT_UDP_PORT, DEFAULT_TCP_PORT
from globalvalues import DEFAULT_CONFIG, DEFAULT_PUBLICKEY

TCP_TIMEOUT = 5

class ServerSender(object):
    """
    Provides ServerSender.send_cpm() for sending UDP packets to the DoseNet
    server.
    """

    def __init__(self,
                 manager=None,
                 network_status=None,
                 address=DEFAULT_HOSTNAME,
                 port=None,
                 config=None,
                 publickey=None,
                 verbosity=1,
                 logfile=None,
                 mode=None,
                 ):
        """
        network_status, config, publickey loaded from manager if not provided.
        address and port take system defaults, although without config and
          publickey, address and port will not be used.
        """

        self.v = verbosity
        if manager and logfile is None:
            set_verbosity(self, logfile=manager.logfile)
        else:
            set_verbosity(self, logfile=logfile)

        self.handle_input(
            manager, mode, port, network_status, config, publickey)

        self.tcp_sender = None
        self.address = address

    def handle_input(
            self, manager, mode, port, network_status, config, publickey):

        # TODO: this stuff is messy. Is there a cleaner way using exceptions?
        if manager is None:
            self.vprint(1, 'ServerSender starting without Manager object')
        self.manager = manager

        try:
            if mode is None:
                self.mode = DEFAULT_SENDER_MODE
            elif mode.lower() == 'udp':
                self.mode = 'udp'
            elif mode.lower() == 'tcp':
                self.mode = 'tcp'
            else:
                raise RuntimeError(
                    'Invalid ServerSender mode (choose TCP or UDP)')
        except AttributeError:
            raise RuntimeError('Invalid ServerSender mode (choose TCP or UDP)')

        if self.mode == 'udp':
            self.port = DEFAULT_UDP_PORT
        elif self.mode == 'tcp':
            self.port = DEFAULT_TCP_PORT

        if network_status is None:
            if manager is None:
                self.vprint(
                    1, 'ServerSender starting without network status object')
                self.network_up = None
            else:
                self.network_up = manager.network_up
        else:
            self.network_up = network_status

        if config is None:
            if manager is None:
                self.vprint(1, 'ServerSender starting without config file')
                self.config = None
            else:
                self.config = manager.config
        else:
            self.config = config

        if publickey is None:
            if manager is None:
                self.vprint(1, 'ServerSender starting without publickey file')
                self.encrypter = None
            elif manager.publickey is None:
                self.vprint(1, 'ServerSender starting without publickey file')
                self.encrypter = None
            else:
                self.encrypter = manager.publickey.encrypter
        else:
            self.encrypter = publickey.encrypter

    def construct_packet(self, cpm, cpm_error, error_code=0):
        """
        Construct the raw packet string. (basically copied from old code)
        """

        c = ','
        try:
            raw_packet = (
                str(self.config.hash) + c +
                str(self.config.ID) + c +
                str(cpm) + c +
                str(cpm_error) + c +
                str(error_code))
        except AttributeError:      # on self.config.hash
            raise MissingFile('Missing or broken Config object')
        else:
            self.vprint(3, 'Constructed packet')
            return raw_packet

    def encrypt_packet(self, raw_packet):
        """Encrypt the raw packet"""

        self.vprint(3, 'Encrypting packet: {}'.format(raw_packet))
        try:
            encrypted = self.encrypter.encrypt_message(raw_packet)[0]
        except AttributeError:
            raise MissingFile('Missing or broken PublicKey object')
        else:
            return encrypted

    def send_data(self, encrypted):
        """
        Send data according to self.mode, and handle common errors
        """

        self.vprint(3, 'Trying to send data by {}'.format(self.mode))
        try:
            if self.mode == 'udp':
                self.send_udp(encrypted)
            elif self.mode == 'tcp':
                self.send_tcp(encrypted)
        except socket.gaierror as e:
            if e[0] == socket.EAI_AGAIN:
                # TCP and UDP
                # network is down, but NetworkStatus didn't notice yet
                # (resolving DNS like dosenet.dhcp.lbl.gov)
                self.vprint(
                    1, 'Failed to send packet! Address resolution error')
                self.network_up.update()
            else:
                self.vprint(1, 'Failed to send packet! Address error: ' +
                            '{}: {}'.format(*e))
        except socket.error as e:
            if e[0] == errno.ECONNREFUSED:
                # TCP
                # server is not accepting connections
                self.vprint(1, 'Failed to send packet! Connection refused')
            elif e[0] == errno.ENETUNREACH:
                # TCP and UDP
                # network is down, but NetworkStatus didn't notice yet
                # (IP like 131.243.51.241)
                self.vprint(
                    1, 'Failed to send packet! Network is unreachable')
                self.network_up.update()
            else:
                # consider handling errno.ECONNABORTED, errno.ECONNRESET
                self.vprint(1, 'Failed to send packet! Socket error: ' +
                            '{}: {}'.format(*e))
        except socket.timeout:
            # TCP
            self.vprint(1, 'Failed to send packet! Socket timeout')
            self.network_up.update()

    def send_udp(self, encrypted):
        """
        Send the encrypted packet over UDP
        """

        self.vprint(3, 'Sending encrypted UDP packet to {}:{}'.format(
            self.address, self.port))
        with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as s:
            s.sendto(encrypted, (self.address, self.port))
            self.vprint(
                3, 'UDP packet sent successfully (no client-side error)')

    def send_tcp(self, encrypted):
        """
        Send the encrypted packet over TCP
        """

        self.vprint(3, 'Sending encrypted TCP packet to {}:{}'.format(
            self.address, self.port))
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
            s.settimeout(TCP_TIMEOUT)   # generous timeout
            s.connect((self.address, self.port))
            s.sendall(encrypted)
            self.vprint(3, 'TCP packet sent successfully')

    def send_cpm(self, cpm, cpm_error, error_code=0):
        """Construct, encrypt, and send the packet"""

        packet = self.construct_packet(cpm, cpm_error, error_code=error_code)
        encrypted = self.encrypt_packet(packet)
        if self.network_up or self.network_up is None:
            self.send_data(encrypted)
        else:
            # TODO: feature add here
            self.vprint(2, 'Network DOWN, not sending packet')

class PacketError(Exception):
    pass

class MissingFile(PacketError):
    pass

def send_test_packets(
        mode=DEFAULT_SENDER_MODE,
        config=DEFAULT_CONFIG,
        publickey=DEFAULT_PUBLICKEY,
        address=DEFAULT_HOSTNAME,
        n=3):
    """
    Send n (default 3) test packets to the DoseNet server.
    """

    sleep_time = 2      # seconds

    try:
        config_obj = Config(config)
    except IOError:
        # file doesn't exist
        config_obj = None
    key_obj = PublicKey(publickey)

    sender = ServerSender(
        mode=mode, address=address,
        config=config_obj, publickey=key_obj, verbosity=3)

    try:
        station_id = config_obj.ID
    except AttributeError:
        station_id = '?'
    raw_packet = 'Test packet from station {} by mode {}'.format(
        station_id, mode)

    encrypted = sender.encrypt_packet(raw_packet)

    for _ in xrange(n):
        sender.send_data(encrypted)
        time.sleep(sleep_time)

if __name__ == '__main__':
    # send test packets
    parser = argparse.ArgumentParser(
        description='Sender for UDP/TCP data packets. ' +
        'Normally called from manager.py. ' +
        'Called directly, it will send 3 test packets to server. ' +
        'Specify udp or tcp on command line.')
    parser.add_argument(
        'mode', choices=['udp', 'tcp'], nargs='?', default='udp',
        help='The network protocol to use in sending test packets')
    args = parser.parse_args()

    send_test_packets(
        mode=args.mode, config=DEFAULT_CONFIG, publickey=DEFAULT_PUBLICKEY)
<|endoftext|>


<commit_before>from twisted.internet import defer
from buildbot.status import mail

from ..factories import TravisSpawnerFactory
from ..travisyml import TravisYml, TravisYmlInvalid

class MailNotifier(mail.MailNotifier):

    def getConfiguration(self, build):
        for step in build.getSteps():
            for log in step.getLogs():
                if log.getName() != '.travis.yml':
                    continue

                config = TravisYml()
                config.parse(log.getText())
                return config
        raise TravisYmlInvalid("No configuration found in branch")

    def isMailNeeded(self, build, results):
        builder = build.getBuilder()
        builder_config = filter(lambda b: b.name == builder.name, self.master.config.builders)[0]

        # This notifier will only generate emails for the "spawner" builds
        if not isinstance(builder_config.factory, TravisSpawnerFactory):
            return False

        # That have valid configuration
        try:
            config = self.getConfiguration(build)
        except TravisYmlInvalid:
            return False

        # And emails are enabled
        if not config.email.enabled:
            return False

        def decide(config):
            if config == "never":
                return False
            elif config == "always":
                return True
            elif config == "change":
                prev = build.getPreviousBuild()
                return prev.getResults() != results

        if results == SUCCESS:
            decide(config.email.success)
        else:
            decide(config.email.failure)

        return False

    def getTravisAddresses(self, build):
        config = self.getConfiguration(build)
        return config.email.addresses

    @defer.inlineCallbacks
    def useLookup(self, build):
        recipients = self.getTravisAddresses(build)
        if not recipients:
            recipients = yield mail.MailNotifier.useLookup(self, build)
        defer.returnValue(recipients)

    @defer.inlineCallbacks
    def useUsers(self, build):
        recipients = self.getTravisAddresses(build)
        if not recipients:
            recipients = yield mail.MailNotifier.useUsers(self, build)
        defer.returnValue(recipients)

<commit_msg>Missing constants<commit_after>from twisted.internet import defer
from buildbot.status import mail
from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION, Results

from ..factories import TravisSpawnerFactory
from ..travisyml import TravisYml, TravisYmlInvalid

class MailNotifier(mail.MailNotifier):

    def getConfiguration(self, build):
        for step in build.getSteps():
            for log in step.getLogs():
                if log.getName() != '.travis.yml':
                    continue

                config = TravisYml()
                config.parse(log.getText())
                return config
        raise TravisYmlInvalid("No configuration found in branch")

    def isMailNeeded(self, build, results):
        builder = build.getBuilder()
        builder_config = filter(lambda b: b.name == builder.name, self.master.config.builders)[0]

        # This notifier will only generate emails for the "spawner" builds
        if not isinstance(builder_config.factory, TravisSpawnerFactory):
            return False

        # That have valid configuration
        try:
            config = self.getConfiguration(build)
        except TravisYmlInvalid:
            return False

        # And emails are enabled
        if not config.email.enabled:
            return False

        def decide(config):
            if config == "never":
                return False
            elif config == "always":
                return True
            elif config == "change":
                prev = build.getPreviousBuild()
                return prev.getResults() != results

        if results == SUCCESS:
            decide(config.email.success)
        else:
            decide(config.email.failure)

        return False

    def getTravisAddresses(self, build):
        config = self.getConfiguration(build)
        return config.email.addresses

    @defer.inlineCallbacks
    def useLookup(self, build):
        recipients = self.getTravisAddresses(build)
        if not recipients:
            recipients = yield mail.MailNotifier.useLookup(self, build)
        defer.returnValue(recipients)

    @defer.inlineCallbacks
    def useUsers(self, build):
        recipients = self.getTravisAddresses(build)
        if not recipients:
            recipients = yield mail.MailNotifier.useUsers(self, build)
        defer.returnValue(recipients)

<|endoftext|>