<commit_before>"""
Directory, which is for utility/non-metric endpoints.
"""

import datetime
import sqlalchemy as s
import pandas as pd
import base64
from augur.util import logger, annotate, add_metrics

@annotate(tag='repo-groups')
def repo_groups(self):
    """
    Returns number of lines changed per author per day

    :param repo_url: the repository's URL
    """
    repoGroupsSQL = s.sql.text("""
        SELECT *
        FROM repo_groups
    """)
    results = pd.read_sql(repoGroupsSQL, self.database)
    return results

@annotate(tag='downloaded-repos')
def downloaded_repos(self):
    """
    Returns all repository names, URLs, and base64 URLs in the facade database
    """
    downloadedReposSQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time ,
            rg_name,
            repo.repo_group_id
        FROM
            repo
            left outer join
            (select repo_id,    COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( * ) as issues_all_time from issues where issues.pull_request IS NULL  group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        order by commits_all_time desc
    """)
    results = pd.read_sql(downloadedReposSQL, self.database)
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    # if self.projects:
    #     results = results[results.project_name.isin(self.projects)]
    if self.projects:
          results = results[results.project_name.isin(self.projects)]

    b64_urls = []
    for i in results.index:
        b64_urls.append(base64.b64encode((results.at[i, 'url']).encode()))
    results['base64_url'] = b64_urls

    return results

@annotate(tag='repos-in-repo-groups')
def repos_in_repo_groups(self, repo_group_id):
    """
    Returns a list of all the repos in a repo_group

    :param repo_group_id: The repository's repo_group_id
    """
    repos_in_repo_groups_SQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time
        FROM
            repo
            left outer join
            (select repo_id, COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE
            repo_groups.repo_group_id = :repo_group_id
        ORDER BY commits_all_time DESC
    """)

    results = pd.read_sql(repos_in_repo_groups_SQL, self.database, params={'repo_group_id': repo_group_id})
    return results

@annotate(tag='get-repo-by-git-name')
def get_repo_by_git_name(self, owner, repo):
    """
    Returns repo id and repo group id by owner and repo

    :param owner: the owner of the repo
    :param repo: the name of the repo
    """
    getRepoSQL = s.sql.text("""
        SELECT repo.repo_id, repo.repo_group_id, rg_name
        FROM repo JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE repo_name = :repo AND repo_path LIKE :owner
        GROUP BY repo_id, rg_name
    """)

    results = pd.read_sql(getRepoSQL, self.database, params={'owner': '%{}_'.format(owner), 'repo': repo,})

    return results

@annotate(tag='get-repo-by-name')
def get_repo_by_name(self, rg_name, repo_name):
    """
    Returns repo id and repo group id by rg_name and repo_name

    :param rg_name: the repo group of the repo
    :param repo_name: the name of the repo
    """

    repoSQL = s.sql.text("""
        SELECT repo_id, repo.repo_group_id, repo_git as url
        FROM repo, repo_groups
        WHERE repo.repo_group_id = repo_groups.repo_group_id
        AND LOWER(rg_name) = LOWER(:rg_name)
        AND LOWER(repo_name) = LOWER(:repo_name)
    """)
    results = pd.read_sql(repoSQL, self.database, params={'rg_name': rg_name, 'repo_name': repo_name})
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    return results

def get_group_by_name(self, rg_name):
    """
    Returns repo group id by repo group name

    :param rg_name:
    """
    groupSQL = s.sql.text("""
        SELECT repo_group_id, rg_name
        FROM repo_groups
        WHERE lower(rg_name) = lower(:rg_name)
    """)
    results = pd.read_sql(groupSQL, self.database, params={'rg_name': rg_name})
    return results

@annotate(tag='dosocs-repos')
def get_repos_for_dosocs(self):
    """ Returns a list of repos along with their repo_id & path """
    get_repos_for_dosocs_SQL = s.sql.text("""
        SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) AS path
        FROM settings a, repo b
        WHERE a.setting='repo_directory'
    """)

    results = pd.read_sql(get_repos_for_dosocs_SQL, self.database)
    return results

@annotate(tag="get-issues")
def get_issues(self, repo_group_id, repo_id=None):
    if not repo_id:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY
            FROM issues,
                issue_events
            WHERE issues.repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id)
            AND issues.issue_id = issue_events.issue_id
            AND issues.pull_request is NULL
            GROUP BY issues.issue_id
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_group_id': repo_group_id})
        return results
    else:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY,
                repo_name
            FROM issues JOIN repo ON issues.repo_id = repo.repo_id, issue_events
            WHERE issues.repo_id = :repo_id
            AND issues.pull_request IS NULL
            AND issues.issue_id = issue_events.issue_id
            GROUP BY issues.issue_id, repo_name
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_id': repo_id})
        return results

@annotate(tag="aggregate-summary")
def aggregate_summary(self, repo_group_id, repo_id=None, begin_date=None, end_date=None):

    if not begin_date:
        begin_date = datetime.datetime.now()
        # Subtract 1 year and leap year check
        try:
            begin_date = begin_date.replace(year=begin_date.year-1)
        except ValueError:
            begin_date = begin_date.replace(year=begin_date.year-1, day=begin_date.day-1)
        begin_date = begin_date.strftime('%Y-%m-%d')
    if not end_date:
        end_date = datetime.datetime.now().strftime('%Y-%m-%d')

    if not repo_id:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id JOIN repo ON issues.repo_id = repo.repo_id
                    WHERE action = 'merged'
                    AND repo_group_id = :repo_group_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits JOIN repo ON commits.repo_id = repo.repo_id
                    WHERE repo_group_id = :repo_group_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_group_id': repo_group_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results
    else:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id
                    WHERE action = 'merged'
                    AND repo_id = :repo_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits
                    WHERE repo_id = :repo_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_id': repo_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results

def create_util_metrics(metrics):
    add_metrics(metrics, __name__)
<commit_msg>Changed default repo_group and repo ordering to their respective names. People love and expect alphabetical things.<commit_after>"""
Directory, which is for utility/non-metric endpoints.
"""

import datetime
import sqlalchemy as s
import pandas as pd
import base64
from augur.util import logger, annotate, add_metrics

@annotate(tag='repo-groups')
def repo_groups(self):
    """
    Returns number of lines changed per author per day

    :param repo_url: the repository's URL
    """
    repoGroupsSQL = s.sql.text("""
        SELECT *
        FROM repo_groups
        ORDER BY rg_name
    """)
    results = pd.read_sql(repoGroupsSQL, self.database)
    return results

@annotate(tag='downloaded-repos')
def downloaded_repos(self):
    """
    Returns all repository names, URLs, and base64 URLs in the facade database
    """
    downloadedReposSQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time ,
            rg_name,
            repo.repo_group_id
        FROM
            repo
            left outer join
            (select repo_id,    COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( * ) as issues_all_time from issues where issues.pull_request IS NULL  group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        order by rg_name
    """)
    results = pd.read_sql(downloadedReposSQL, self.database)
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    # if self.projects:
    #     results = results[results.project_name.isin(self.projects)]
    if self.projects:
          results = results[results.project_name.isin(self.projects)]

    b64_urls = []
    for i in results.index:
        b64_urls.append(base64.b64encode((results.at[i, 'url']).encode()))
    results['base64_url'] = b64_urls

    return results

@annotate(tag='repos-in-repo-groups')
def repos_in_repo_groups(self, repo_group_id):
    """
    Returns a list of all the repos in a repo_group

    :param repo_group_id: The repository's repo_group_id
    """
    repos_in_repo_groups_SQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time
        FROM
            repo
            left outer join
            (select repo_id, COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE
            repo_groups.repo_group_id = :repo_group_id
        ORDER BY commits_all_time DESC
    """)

    results = pd.read_sql(repos_in_repo_groups_SQL, self.database, params={'repo_group_id': repo_group_id})
    return results

@annotate(tag='get-repo-by-git-name')
def get_repo_by_git_name(self, owner, repo):
    """
    Returns repo id and repo group id by owner and repo

    :param owner: the owner of the repo
    :param repo: the name of the repo
    """
    getRepoSQL = s.sql.text("""
        SELECT repo.repo_id, repo.repo_group_id, rg_name
        FROM repo JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE repo_name = :repo AND repo_path LIKE :owner
        GROUP BY repo_id, rg_name
    """)

    results = pd.read_sql(getRepoSQL, self.database, params={'owner': '%{}_'.format(owner), 'repo': repo,})

    return results

@annotate(tag='get-repo-by-name')
def get_repo_by_name(self, rg_name, repo_name):
    """
    Returns repo id and repo group id by rg_name and repo_name

    :param rg_name: the repo group of the repo
    :param repo_name: the name of the repo
    """

    repoSQL = s.sql.text("""
        SELECT repo_id, repo.repo_group_id, repo_git as url
        FROM repo, repo_groups
        WHERE repo.repo_group_id = repo_groups.repo_group_id
        AND LOWER(rg_name) = LOWER(:rg_name)
        AND LOWER(repo_name) = LOWER(:repo_name)
    """)
    results = pd.read_sql(repoSQL, self.database, params={'rg_name': rg_name, 'repo_name': repo_name})
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    return results

def get_group_by_name(self, rg_name):
    """
    Returns repo group id by repo group name

    :param rg_name:
    """
    groupSQL = s.sql.text("""
        SELECT repo_group_id, rg_name
        FROM repo_groups
        WHERE lower(rg_name) = lower(:rg_name)
    """)
    results = pd.read_sql(groupSQL, self.database, params={'rg_name': rg_name})
    return results

@annotate(tag='dosocs-repos')
def get_repos_for_dosocs(self):
    """ Returns a list of repos along with their repo_id & path """
    get_repos_for_dosocs_SQL = s.sql.text("""
        SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) AS path
        FROM settings a, repo b
        WHERE a.setting='repo_directory'
    """)

    results = pd.read_sql(get_repos_for_dosocs_SQL, self.database)
    return results

@annotate(tag="get-issues")
def get_issues(self, repo_group_id, repo_id=None):
    if not repo_id:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY
            FROM issues,
                issue_events
            WHERE issues.repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id)
            AND issues.issue_id = issue_events.issue_id
            AND issues.pull_request is NULL
            GROUP BY issues.issue_id
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_group_id': repo_group_id})
        return results
    else:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY,
                repo_name
            FROM issues JOIN repo ON issues.repo_id = repo.repo_id, issue_events
            WHERE issues.repo_id = :repo_id
            AND issues.pull_request IS NULL
            AND issues.issue_id = issue_events.issue_id
            GROUP BY issues.issue_id, repo_name
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_id': repo_id})
        return results

@annotate(tag="aggregate-summary")
def aggregate_summary(self, repo_group_id, repo_id=None, begin_date=None, end_date=None):

    if not begin_date:
        begin_date = datetime.datetime.now()
        # Subtract 1 year and leap year check
        try:
            begin_date = begin_date.replace(year=begin_date.year-1)
        except ValueError:
            begin_date = begin_date.replace(year=begin_date.year-1, day=begin_date.day-1)
        begin_date = begin_date.strftime('%Y-%m-%d')
    if not end_date:
        end_date = datetime.datetime.now().strftime('%Y-%m-%d')

    if not repo_id:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id JOIN repo ON issues.repo_id = repo.repo_id
                    WHERE action = 'merged'
                    AND repo_group_id = :repo_group_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits JOIN repo ON commits.repo_id = repo.repo_id
                    WHERE repo_group_id = :repo_group_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_group_id': repo_group_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results
    else:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id
                    WHERE action = 'merged'
                    AND repo_id = :repo_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits
                    WHERE repo_id = :repo_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_id': repo_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results

def create_util_metrics(metrics):
    add_metrics(metrics, __name__)
<|endoftext|>


<commit_before>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

import unittest2 as unittest

from autobahn.util import IdGenerator

class TestIdGenerator(unittest.TestCase):
    def test_idgenerator_is_generator(self):
        "IdGenerator follows the generator protocol"
        g = IdGenerator()
        self.assertEqual(0, next(g))
        self.assertEqual(1, next(g))

<commit_msg>fix pep8<commit_after>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

import unittest2 as unittest

from autobahn.util import IdGenerator

class TestIdGenerator(unittest.TestCase):
    def test_idgenerator_is_generator(self):
        "IdGenerator follows the generator protocol"
        g = IdGenerator()
        self.assertEqual(0, next(g))
        self.assertEqual(1, next(g))
<|endoftext|>


<commit_before>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

import traceback

from autobahn.websocket import protocol
from autobahn.websocket.types import ConnectionDeny
from autobahn.wamp.interfaces import ITransport
from autobahn.wamp.exception import ProtocolError, SerializationError, TransportLost

__all__ = ('WampWebSocketServerProtocol',
           'WampWebSocketClientProtocol',
           'WampWebSocketServerFactory',
           'WampWebSocketClientFactory')

class WampWebSocketProtocol(object):
    """
    Base class for WAMP-over-WebSocket transport mixins.
    """

    _session = None  # default; self.session is set in onOpen

    def _bailout(self, code, reason=None):
        self.log.debug('Failing WAMP-over-WebSocket transport: code={code}, reason="{reason}"', code=code, reason=reason)
        self.failConnection(code, reason)

    def onOpen(self):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onOpen`
        """
        # WebSocket connection established. Now let the user WAMP session factory
        # create a new WAMP session and fire off session open callback.
        try:
            self._session = self.factory._factory()
            self._session.onOpen(self)
        except Exception as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Internal Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_INTERNAL_ERROR, reason=reason)

    def onClose(self, wasClean, code, reason):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onClose`
        """
        # WAMP session might never have been established in the first place .. guard this!
        if self._session is not None:
            # WebSocket connection lost - fire off the WAMP
            # session close callback
            # noinspection PyBroadException
            try:
                self.log.debug('WAMP-over-WebSocket transport lost: wasClean={wasClean}, code={code}, reason="{reason}"', wasClean=wasClean, code=code, reason=reason)
                self._session.onClose(wasClean)
            except Exception:
                self.log.critical(traceback.format_exc())
            self._session = None

    def onMessage(self, payload, isBinary):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onMessage`
        """
        try:
            for msg in self._serializer.unserialize(payload, isBinary):
                self.log.trace("WAMP RECV: message={message}, session={session}, authid={authid}", authid=self._session._authid, session=self._session._session_id, message=msg)
                self._session.onMessage(msg)

        except ProtocolError as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Protocol Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_PROTOCOL_ERROR, reason=reason)

        except Exception as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Internal Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_INTERNAL_ERROR, reason=reason)

    def send(self, msg):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.send`
        """
        if self.isOpen():
            try:
                self.log.trace("WAMP SEND: message={message}, session={session}, authid={authid}", authid=self._session._authid, session=self._session._session_id, message=msg)
                payload, isBinary = self._serializer.serialize(msg)
            except Exception as e:
                self.log.failure("WAMP message serialization error")
                # all exceptions raised from above should be serialization errors ..
                raise SerializationError(u"WAMP message serialization error: {0}".format(e))
            else:
                self.sendMessage(payload, isBinary)
        else:
            raise TransportLost()

    def isOpen(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.isOpen`
        """
        return self._session is not None

    def close(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.close`
        """
        if self.isOpen():
            self.sendClose(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_NORMAL)
        else:
            raise TransportLost()

    def abort(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.abort`
        """
        if self.isOpen():
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_GOING_AWAY)
        else:
            raise TransportLost()

ITransport.register(WampWebSocketProtocol)

def parseSubprotocolIdentifier(subprotocol):
    try:
        s = subprotocol.split(u'.')
        if s[0] != u'wamp':
            raise Exception(u'WAMP WebSocket subprotocol identifier must start with "wamp", not "{}"'.format(s[0]))
        version = int(s[1])
        serializerId = u'.'.join(s[2:])
        return version, serializerId
    except:
        return None, None

class WampWebSocketServerProtocol(WampWebSocketProtocol):
    """
    Mixin for WAMP-over-WebSocket server transports.
    """

    STRICT_PROTOCOL_NEGOTIATION = True

    def onConnect(self, request):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onConnect`
        """
        headers = {}
        for subprotocol in request.protocols:
            version, serializerId = parseSubprotocolIdentifier(subprotocol)
            if version == 2 and serializerId in self.factory._serializers.keys():
                self._serializer = self.factory._serializers[serializerId]
                return subprotocol, headers

        if self.STRICT_PROTOCOL_NEGOTIATION:
            raise ConnectionDeny(ConnectionDeny.BAD_REQUEST, u'This server only speaks WebSocket subprotocols {}'.format(u', '.join(self.factory.protocols)))
        else:
            # assume wamp.2.json
            self._serializer = self.factory._serializers[u'json']
            return None, headers

class WampWebSocketClientProtocol(WampWebSocketProtocol):
    """
    Mixin for WAMP-over-WebSocket client transports.
    """

    STRICT_PROTOCOL_NEGOTIATION = True

    def onConnect(self, response):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onConnect`
        """
        if response.protocol not in self.factory.protocols:
            if self.STRICT_PROTOCOL_NEGOTIATION:
                raise Exception(u'The server does not speak any of the WebSocket subprotocols {} we requested.'.format(u', '.join(self.factory.protocols)))
            else:
                # assume wamp.2.json
                serializerId = u'json'
        else:
            version, serializerId = parseSubprotocolIdentifier(response.protocol)

        self._serializer = self.factory._serializers[serializerId]

class WampWebSocketFactory(object):
    """
    Base class for WAMP-over-WebSocket transport factory mixins.
    """

    def __init__(self, factory, serializers=None):
        """
        Ctor.

        :param factory: A callable that produces instances that implement
           :class:`autobahn.wamp.interfaces.ITransportHandler`
        :type factory: callable
        :param serializers: A list of WAMP serializers to use (or None for default
           serializers). Serializers must implement
           :class:`autobahn.wamp.interfaces.ISerializer`.
        :type serializers: list
        """
        if not callable(factory):
            raise Exception(u'factory must be a callable')
        self._factory = factory

        if serializers is None:
            serializers = []

            # try CBOR WAMP serializer
            try:
                from autobahn.wamp.serializer import CBORSerializer
                serializers.append(CBORSerializer(batched=True))
                serializers.append(CBORSerializer())
            except ImportError:
                pass

            # try MsgPack WAMP serializer
            try:
                from autobahn.wamp.serializer import MsgPackSerializer
                serializers.append(MsgPackSerializer(batched=True))
                serializers.append(MsgPackSerializer())
            except ImportError:
                pass

            # try JSON WAMP serializer
            try:
                from autobahn.wamp.serializer import JsonSerializer
                serializers.append(JsonSerializer(batched=True))
                serializers.append(JsonSerializer())
            except ImportError:
                pass

            if not serializers:
                raise Exception(u'Could not import any WAMP serializer')

        self._serializers = {}
        for ser in serializers:
            self._serializers[ser.SERIALIZER_ID] = ser

        self._protocols = [u'wamp.2.{}'.format(ser.SERIALIZER_ID) for ser in serializers]

class WampWebSocketServerFactory(WampWebSocketFactory):
    """
    Mixin for WAMP-over-WebSocket server transport factories.
    """

class WampWebSocketClientFactory(WampWebSocketFactory):
    """
    Mixin for WAMP-over-WebSocket client transport factories.
    """
<commit_msg>Fix error handling logger method.<commit_after>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

import traceback

from autobahn.websocket import protocol
from autobahn.websocket.types import ConnectionDeny
from autobahn.wamp.interfaces import ITransport
from autobahn.wamp.exception import ProtocolError, SerializationError, TransportLost

__all__ = ('WampWebSocketServerProtocol',
           'WampWebSocketClientProtocol',
           'WampWebSocketServerFactory',
           'WampWebSocketClientFactory')

class WampWebSocketProtocol(object):
    """
    Base class for WAMP-over-WebSocket transport mixins.
    """

    _session = None  # default; self.session is set in onOpen

    def _bailout(self, code, reason=None):
        self.log.debug('Failing WAMP-over-WebSocket transport: code={code}, reason="{reason}"', code=code, reason=reason)
        self.failConnection(code, reason)

    def onOpen(self):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onOpen`
        """
        # WebSocket connection established. Now let the user WAMP session factory
        # create a new WAMP session and fire off session open callback.
        try:
            self._session = self.factory._factory()
            self._session.onOpen(self)
        except Exception as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Internal Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_INTERNAL_ERROR, reason=reason)

    def onClose(self, wasClean, code, reason):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onClose`
        """
        # WAMP session might never have been established in the first place .. guard this!
        if self._session is not None:
            # WebSocket connection lost - fire off the WAMP
            # session close callback
            # noinspection PyBroadException
            try:
                self.log.debug('WAMP-over-WebSocket transport lost: wasClean={wasClean}, code={code}, reason="{reason}"', wasClean=wasClean, code=code, reason=reason)
                self._session.onClose(wasClean)
            except Exception:
                self.log.critical(traceback.format_exc())
            self._session = None

    def onMessage(self, payload, isBinary):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onMessage`
        """
        try:
            for msg in self._serializer.unserialize(payload, isBinary):
                self.log.trace("WAMP RECV: message={message}, session={session}, authid={authid}", authid=self._session._authid, session=self._session._session_id, message=msg)
                self._session.onMessage(msg)

        except ProtocolError as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Protocol Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_PROTOCOL_ERROR, reason=reason)

        except Exception as e:
            self.log.critical(traceback.format_exc())
            reason = u'WAMP Internal Error ({0})'.format(e)
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_INTERNAL_ERROR, reason=reason)

    def send(self, msg):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.send`
        """
        if self.isOpen():
            try:
                self.log.trace("WAMP SEND: message={message}, session={session}, authid={authid}", authid=self._session._authid, session=self._session._session_id, message=msg)
                payload, isBinary = self._serializer.serialize(msg)
            except Exception as e:
                self.log.error("WAMP message serialization error")
                # all exceptions raised from above should be serialization errors ..
                raise SerializationError(u"WAMP message serialization error: {0}".format(e))
            else:
                self.sendMessage(payload, isBinary)
        else:
            raise TransportLost()

    def isOpen(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.isOpen`
        """
        return self._session is not None

    def close(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.close`
        """
        if self.isOpen():
            self.sendClose(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_NORMAL)
        else:
            raise TransportLost()

    def abort(self):
        """
        Implements :func:`autobahn.wamp.interfaces.ITransport.abort`
        """
        if self.isOpen():
            self._bailout(protocol.WebSocketProtocol.CLOSE_STATUS_CODE_GOING_AWAY)
        else:
            raise TransportLost()

ITransport.register(WampWebSocketProtocol)

def parseSubprotocolIdentifier(subprotocol):
    try:
        s = subprotocol.split(u'.')
        if s[0] != u'wamp':
            raise Exception(u'WAMP WebSocket subprotocol identifier must start with "wamp", not "{}"'.format(s[0]))
        version = int(s[1])
        serializerId = u'.'.join(s[2:])
        return version, serializerId
    except:
        return None, None

class WampWebSocketServerProtocol(WampWebSocketProtocol):
    """
    Mixin for WAMP-over-WebSocket server transports.
    """

    STRICT_PROTOCOL_NEGOTIATION = True

    def onConnect(self, request):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onConnect`
        """
        headers = {}
        for subprotocol in request.protocols:
            version, serializerId = parseSubprotocolIdentifier(subprotocol)
            if version == 2 and serializerId in self.factory._serializers.keys():
                self._serializer = self.factory._serializers[serializerId]
                return subprotocol, headers

        if self.STRICT_PROTOCOL_NEGOTIATION:
            raise ConnectionDeny(ConnectionDeny.BAD_REQUEST, u'This server only speaks WebSocket subprotocols {}'.format(u', '.join(self.factory.protocols)))
        else:
            # assume wamp.2.json
            self._serializer = self.factory._serializers[u'json']
            return None, headers

class WampWebSocketClientProtocol(WampWebSocketProtocol):
    """
    Mixin for WAMP-over-WebSocket client transports.
    """

    STRICT_PROTOCOL_NEGOTIATION = True

    def onConnect(self, response):
        """
        Callback from :func:`autobahn.websocket.interfaces.IWebSocketChannel.onConnect`
        """
        if response.protocol not in self.factory.protocols:
            if self.STRICT_PROTOCOL_NEGOTIATION:
                raise Exception(u'The server does not speak any of the WebSocket subprotocols {} we requested.'.format(u', '.join(self.factory.protocols)))
            else:
                # assume wamp.2.json
                serializerId = u'json'
        else:
            version, serializerId = parseSubprotocolIdentifier(response.protocol)

        self._serializer = self.factory._serializers[serializerId]

class WampWebSocketFactory(object):
    """
    Base class for WAMP-over-WebSocket transport factory mixins.
    """

    def __init__(self, factory, serializers=None):
        """
        Ctor.

        :param factory: A callable that produces instances that implement
           :class:`autobahn.wamp.interfaces.ITransportHandler`
        :type factory: callable
        :param serializers: A list of WAMP serializers to use (or None for default
           serializers). Serializers must implement
           :class:`autobahn.wamp.interfaces.ISerializer`.
        :type serializers: list
        """
        if not callable(factory):
            raise Exception(u'factory must be a callable')
        self._factory = factory

        if serializers is None:
            serializers = []

            # try CBOR WAMP serializer
            try:
                from autobahn.wamp.serializer import CBORSerializer
                serializers.append(CBORSerializer(batched=True))
                serializers.append(CBORSerializer())
            except ImportError:
                pass

            # try MsgPack WAMP serializer
            try:
                from autobahn.wamp.serializer import MsgPackSerializer
                serializers.append(MsgPackSerializer(batched=True))
                serializers.append(MsgPackSerializer())
            except ImportError:
                pass

            # try JSON WAMP serializer
            try:
                from autobahn.wamp.serializer import JsonSerializer
                serializers.append(JsonSerializer(batched=True))
                serializers.append(JsonSerializer())
            except ImportError:
                pass

            if not serializers:
                raise Exception(u'Could not import any WAMP serializer')

        self._serializers = {}
        for ser in serializers:
            self._serializers[ser.SERIALIZER_ID] = ser

        self._protocols = [u'wamp.2.{}'.format(ser.SERIALIZER_ID) for ser in serializers]

class WampWebSocketServerFactory(WampWebSocketFactory):
    """
    Mixin for WAMP-over-WebSocket server transport factories.
    """

class WampWebSocketClientFactory(WampWebSocketFactory):
    """
    Mixin for WAMP-over-WebSocket client transport factories.
    """
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
   Condor-related common utilities and library for AutoPyFactory.
   Focussed on properly processing output of condor_q -xml and condor_status -xml and converting
   to native Python data structures.

"""
import commands
import datetime
import logging
import os
import re
import signal
import subprocess
import sys
import threading
import time
import traceback
import xml.dom.minidom

from autopyfactory.apfexceptions import ConfigFailure, CondorVersionFailure
from autopyfactory.info import JobInfo
from autopyfactory import utils as utils

from pprint import pprint
from Queue import Queue

import htcondor
import classad
import copy

#############################################################################
#              APF API to query methods
#############################################################################

def querycondorlib(remotecollector=None, remoteschedd=None, extra_attributes=[], queueskey='match_apf_queue'):
    """
    queries condor to get a list of ClassAds objects
    We query for a few specific ClassAd attributes
    (faster than getting everything)

    remotecollector and remoteschedd
    are passed when querying a remote HTCondor pool
    They are the equivalent to -pool and -name input
    options to CLI condor_q

    extra_attributes are classads needed other than 'jobstatus'
    """

    log = logging.getLogger('autopyfactory')
    log.debug("Starting with values remotecollector=%s, remoteschedd=%s, extra_attributes=%s, queueskey=%s" %(remotecollector, remoteschedd, extra_attributes, queueskey))

    list_attrs = [queueskey, 'jobstatus']
    list_attrs += extra_attributes
    out = _querycondorlib(list_attrs, remotecollector, remoteschedd)
    ###out = _aggregateinfolib(out, queueskey, 'jobstatus')
    from mappings import JobStatusAnalyzer
    jobstatusanalyzer = JobStatusAnalyzer()
    out = _aggregateinfolib(out, queueskey, [jobstatusanalyzer])
    log.debug(out)
    return out

def condorhistorylib( attributes, constraints=[]):
    default_attributes=['match_apf_queue', 'jobstatus', 'enteredcurrentstatus', 'remotewallclocktime','qdate']
    for da in default_attributes:
        if da not in attributes:
            attributes.append(da)
    logging.debug('history called with attributes: %s' % attributes)
    return _condorhistorylib( attributes, constraints)

#############################################################################
#              condor python methods
#############################################################################

def _querycondorlib(attributes, remotecollector=None, remoteschedd=None):
    '''
    Returns a list of ClassAd objects.
    '''
    # NOTE:
    # when remotecollector has a valid value,
    # then remoteschedd must have a valid value too

    log = logging.getLogger('autopyfactory')
    log.debug("Starting with values attributes=%s, remotecollector=%s, remoteschedd=%s" %(attributes, remotecollector, remoteschedd))
    if remotecollector:
        # FIXME: to be tested
        log.debug("querying remote pool %s" %remotecollector)
        collector = htcondor.Collector(remotecollector)
        scheddAd = collector.locate(htcondor.DaemonTypes.Schedd, remoteschedd)
        schedd = htcondor.Schedd(scheddAd)
    else:
        schedd = htcondor.Schedd() # Defaults to the local schedd.

    out = schedd.query('true', attributes)
    log.debug(out)
    return out

def _condorhistorylib( attributes, constraints):
    schedd = htcondor.Schedd()
    if len(constraints) > 1:
        condor_constraint_expr = " && ".join(constraints)
        history = schedd.history(condor_constraint_expr, attributes, 0)
    else:
        history = schedd.history( "true" , attributes, 0)
    history = list(history)
    return history

def querystatuslib():
    """
    Equivalent to condor_status
    We query for a few specific ClassAd attributes
    (faster than getting everything)
    Output of collector.query(htcondor.AdTypes.Startd) looks like
     [
      [ Name = "slot1@mysite.net"; Activity = "Idle"; MyType = "Machine"; TargetType = "Job"; State = "Unclaimed"; CurrentTime = time() ],
      [ Name = "slot2@mysite.net"; Activity = "Idle"; MyType = "Machine"; TargetType = "Job"; State = "Unclaimed"; CurrentTime = time() ]
     ]
    """
    # We only want to try to import if we are actually using the call...
    # Later on we will need to handle Condor version >7.9.4 and <7.9.4
    #
    collector = htcondor.Collector()
    list_attrs = ['Name', 'State', 'Activity']
    outlist = collector.query(htcondor.AdTypes.Startd, 'true', list_attrs)
    return outlist

#############################################################################
#              parse and aggregate outputs
#############################################################################

def _aggregateinfolib(input, primary_key='match_apf_queue', analyzers=[]):
    # input is a list of job classads
    # analyzers is a list of mappings.BaseAnalyzer objects
    # output is a dict[primary_key] [secondary_key] [value] = # of jobs with that value

    queues = {}

    for job in input:
        if not primary_key in job.keys():
            # This job is not managed by APF. Ignore...
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {}

        for analyzer in analyzers:
            label = analyzer.getlabel()
            if label not in queues[apfqname].keys():
                queues[apfqname][label] = {}
            value = analyzer.analyze(job)
            if value != None:
                if value not in queues[apfqname][label].keys():
                    queues[apfqname][label][value] = 0
                queues[apfqname][label][value] += 1

    return queues

def _aggregatehistoryinfolib(jobs, primary_key='match_apf_queue', queues=None, analyzers=[]):

    if not queues:
        queues = {}
    else:
        queues = queues

    for job in jobs:
        if not primary_key in job:
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {'total':0, 'short':0}

        for analyzer in analyzers:
            out = analyzer.analyze(job)
            if out is not None:
                queues[apfqname]['total'] += 1
                if out is False:
                    queues[apfqname]['short'] += 1

    return queues

def filtercondorhistorylib(history, constraints=[]):

    # contraints example ['JobStatus == 4', 'RemoteWallClockTime < 120']

    out = []
    for job in history:
        if _matches_constraints(job, constraints):
            out.append(job)
    return out

def _matches_constraints(ad, constraints):
    constraint_expression = " && ".join( ["TARGET." + i for i in constraints])
    return _matches_constraint_expr(ad, constraint_expression)

def _matches_constraint_expr(ad, constraint_expression):
    req_ad = classad.ClassAd()
    req_ad['Requirements'] = classad.ExprTree(constraint_expression)
    return ad.matches(req_ad)

# this is a temp solution to add running jobs to output of condor_history
def _aggregatecondorqinfolib(jobs, primary_key='match_apf_queue', queues=None, analyzers=[]):

    if not queues:
        queues = {}
    else:
        queues = queues

    for job in jobs:
        if not primary_key in job:
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {'total':0}

        for analyzer in analyzers:
            out = analyzer.analyze(job)
            if out is not None:
                if out is True:
                    queues[apfqname]['total'] += 1

    return queues

##############################################################################

def test1():
    infodict = getJobInfo()
    ec2jobs = infodict['BNL_CLOUD-ec2-spot']
    #pprint(ec2jobs)

    startds = getStartdInfoByEC2Id()
    print(startds)
<commit_msg>more reasonable names<commit_after>#!/usr/bin/env python
"""
   Condor-related common utilities and library for AutoPyFactory.
   Focussed on properly processing output of condor_q -xml and condor_status -xml and converting
   to native Python data structures.

"""
import commands
import datetime
import logging
import os
import re
import signal
import subprocess
import sys
import threading
import time
import traceback
import xml.dom.minidom

from autopyfactory.apfexceptions import ConfigFailure, CondorVersionFailure
from autopyfactory.info import JobInfo
from autopyfactory import utils as utils

from pprint import pprint
from Queue import Queue

import htcondor
import classad
import copy

#############################################################################
#              APF interface to query methods
#############################################################################

def querycondorlib(remotecollector=None, remoteschedd=None, extra_attributes=[], queueskey='match_apf_queue'):
    """
    queries condor to get a list of ClassAds objects
    We query for a few specific ClassAd attributes
    (faster than getting everything)

    remotecollector and remoteschedd
    are passed when querying a remote HTCondor pool
    They are the equivalent to -pool and -name input
    options to CLI condor_q

    extra_attributes are classads needed other than 'jobstatus'
    """

    log = logging.getLogger('autopyfactory')
    log.debug("Starting with values remotecollector=%s, remoteschedd=%s, extra_attributes=%s, queueskey=%s" %(remotecollector, remoteschedd, extra_attributes, queueskey))

    list_attrs = [queueskey, 'jobstatus']
    list_attrs += extra_attributes
    out = condor_q(list_attrs, remotecollector, remoteschedd)
    ###out = _aggregateinfolib(out, queueskey, 'jobstatus')
    from mappings import JobStatusAnalyzer
    jobstatusanalyzer = JobStatusAnalyzer()
    out = _aggregateinfolib(out, queueskey, [jobstatusanalyzer])
    log.debug(out)
    return out

def condorhistorylib( attributes, constraints=[]):
    default_attributes=['match_apf_queue', 'jobstatus', 'enteredcurrentstatus', 'remotewallclocktime','qdate']
    for da in default_attributes:
        if da not in attributes:
            attributes.append(da)
    logging.debug('history called with attributes: %s' % attributes)
    return condor_history( attributes, constraints)

#############################################################################
#              condor python methods
#FIXME:
#   if one day we want to make these methods a separately library
#   then most probably they cannot have an 'autopyfactory' logger
#############################################################################

def condor_q(attributes, remotecollector=None, remoteschedd=None):
    '''
    Returns a list of ClassAd objects.
    '''
    # NOTE:
    # when remotecollector has a valid value,
    # then remoteschedd must have a valid value too

    log = logging.getLogger('autopyfactory')
    log.debug("Starting with values attributes=%s, remotecollector=%s, remoteschedd=%s" %(attributes, remotecollector, remoteschedd))
    if remotecollector:
        # FIXME: to be tested
        log.debug("querying remote pool %s" %remotecollector)
        collector = htcondor.Collector(remotecollector)
        scheddAd = collector.locate(htcondor.DaemonTypes.Schedd, remoteschedd)
        schedd = htcondor.Schedd(scheddAd)
    else:
        schedd = htcondor.Schedd() # Defaults to the local schedd.

    out = schedd.query('true', attributes)
    log.debug(out)
    return out

def condor_history( attributes, constraints):
    schedd = htcondor.Schedd()
    if len(constraints) > 1:
        condor_constraint_expr = " && ".join(constraints)
        history = schedd.history(condor_constraint_expr, attributes, 0)
    else:
        history = schedd.history( "true" , attributes, 0)
    history = list(history)
    return history

def condor_status():
    """
    Equivalent to condor_status
    We query for a few specific ClassAd attributes
    (faster than getting everything)
    Output of collector.query(htcondor.AdTypes.Startd) looks like
     [
      [ Name = "slot1@mysite.net"; Activity = "Idle"; MyType = "Machine"; TargetType = "Job"; State = "Unclaimed"; CurrentTime = time() ],
      [ Name = "slot2@mysite.net"; Activity = "Idle"; MyType = "Machine"; TargetType = "Job"; State = "Unclaimed"; CurrentTime = time() ]
     ]
    """
    # We only want to try to import if we are actually using the call...
    # Later on we will need to handle Condor version >7.9.4 and <7.9.4
    #
    collector = htcondor.Collector()
    list_attrs = ['Name', 'State', 'Activity']
    outlist = collector.query(htcondor.AdTypes.Startd, 'true', list_attrs)
    return outlist

#############################################################################
#              parse and aggregate outputs
#############################################################################

def _aggregateinfolib(input, primary_key='match_apf_queue', analyzers=[]):
    # input is a list of job classads
    # analyzers is a list of mappings.BaseAnalyzer objects
    # output is a dict[primary_key] [secondary_key] [value] = # of jobs with that value

    queues = {}

    for job in input:
        if not primary_key in job.keys():
            # This job is not managed by APF. Ignore...
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {}

        for analyzer in analyzers:
            label = analyzer.getlabel()
            if label not in queues[apfqname].keys():
                queues[apfqname][label] = {}
            value = analyzer.analyze(job)
            if value != None:
                if value not in queues[apfqname][label].keys():
                    queues[apfqname][label][value] = 0
                queues[apfqname][label][value] += 1

    return queues

def _aggregatehistoryinfolib(jobs, primary_key='match_apf_queue', queues=None, analyzers=[]):

    if not queues:
        queues = {}
    else:
        queues = queues

    for job in jobs:
        if not primary_key in job:
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {'total':0, 'short':0}

        for analyzer in analyzers:
            out = analyzer.analyze(job)
            if out is not None:
                queues[apfqname]['total'] += 1
                if out is False:
                    queues[apfqname]['short'] += 1

    return queues

def filtercondorhistorylib(history, constraints=[]):

    # contraints example ['JobStatus == 4', 'RemoteWallClockTime < 120']

    out = []
    for job in history:
        if _matches_constraints(job, constraints):
            out.append(job)
    return out

def _matches_constraints(ad, constraints):
    constraint_expression = " && ".join( ["TARGET." + i for i in constraints])
    return _matches_constraint_expr(ad, constraint_expression)

def _matches_constraint_expr(ad, constraint_expression):
    req_ad = classad.ClassAd()
    req_ad['Requirements'] = classad.ExprTree(constraint_expression)
    return ad.matches(req_ad)

# this is a temp solution to add running jobs to output of condor_history
def _aggregatecondorqinfolib(jobs, primary_key='match_apf_queue', queues=None, analyzers=[]):

    if not queues:
        queues = {}
    else:
        queues = queues

    for job in jobs:
        if not primary_key in job:
            continue

        apfqname = str(job[primary_key])
        if apfqname not in queues.keys():
            queues[apfqname] = {'total':0}

        for analyzer in analyzers:
            out = analyzer.analyze(job)
            if out is not None:
                if out is True:
                    queues[apfqname]['total'] += 1

    return queues

##############################################################################

def test1():
    infodict = getJobInfo()
    ec2jobs = infodict['BNL_CLOUD-ec2-spot']
    #pprint(ec2jobs)

    startds = getStartdInfoByEC2Id()
    print(startds)
<|endoftext|>


<commit_before># Standard library
import sys
import importlib

# Pyblish libraries
import pyblish.api

# Host libraries
import hou

# Local libraries
from . import lib
from ..lib import logger
from avalon import api, schema

self = sys.modules[__name__]
self._has_been_setup = False
self._parent = None
self._events = dict()

AVALON_CONTAINERS = "/obj/AVALON_CONTAINERS"
IS_HEADLESS = not hasattr(hou, "ui")

def install(config):
    """Setup integration
    Register plug-ins and integrate into the host

    """

    print("Registering callbacks")
    _register_callbacks()

    pyblish.api.register_host("houdini")
    pyblish.api.register_host("hython")
    pyblish.api.register_host("hpython")

    self._has_been_setup = True

    config = find_host_config(config)
    if hasattr(config, "install"):
        config.install()

def uninstall(config):
    """Uninstall Houdini-specific functionality of avalon-core.

    This function is called automatically on calling `api.uninstall()`.

    Args:
        config: configuration module

    """

    config = find_host_config(config)
    if hasattr(config, "uninstall"):
        config.uninstall()

    pyblish.api.deregister_host("hython")
    pyblish.api.deregister_host("hpython")
    pyblish.api.deregister_host("houdini")

def find_host_config(config):
    config_name = config.__name__
    try:
        config = importlib.import_module(config_name + ".houdini")
    except ImportError as exc:
        if str(exc) != "No module name {}".format(config_name + ".houdini"):
            raise
        config = None

    return config

def reload_pipeline(*args):
    """Attempt to reload pipeline at run-time.

    CAUTION: This is primarily for development and debugging purposes.

    """

    import importlib

    api.uninstall()

    for module in ("avalon.io",
                   "avalon.lib",
                   "avalon.pipeline",

                   "avalon.houdini.pipeline",
                   "avalon.houdini.lib",
                   "avalon.tools.loader.app",
                   "avalon.tools.creator.app",
                   "avalon.tools.manager.app",

                   # NOTE(marcus): These have circular depenendencies
                   #               that is preventing reloadability
                   # "avalon.tools.cbloader.delegates",
                   # "avalon.tools.cbloader.model",
                   # "avalon.tools.cbloader.widgets",
                   # "avalon.tools.cbloader.app",
                   # "avalon.tools.cbsceneinventory.model",
                   # "avalon.tools.cbsceneinventory.proxy",
                   # "avalon.tools.cbsceneinventory.app",
                   # "avalon.tools.projectmanager.dialogs",
                   # "avalon.tools.projectmanager.lib",
                   # "avalon.tools.projectmanager.model",
                   # "avalon.tools.projectmanager.style",
                   # "avalon.tools.projectmanager.widget",
                   # "avalon.tools.projectmanager.app",

                   "avalon.api",
                   "avalon.tools",
                   "avalon.houdini"):
        module = importlib.import_module(module)
        reload(module)

    self._parent = {hou.ui.mainQtWindow().objectName(): hou.ui.mainQtWindow()}

    import avalon.houdini
    api.install(avalon.houdini)

def _discover_gui():
    """Return the most desirable of the currently registered GUIs"""

    # Prefer last registered
    guis = reversed(pyblish.api.registered_guis())

    for gui in guis:
        try:
            gui = __import__(gui).show
        except (ImportError, AttributeError):
            continue
        else:
            return gui

def teardown():
    """Remove integration"""
    if not self._has_been_setup:
        return

    self._has_been_setup = False
    print("pyblish: Integration torn down successfully")

def containerise(name,
                 namespace,
                 nodes,
                 context,
                 loader=None,
                 suffix=""):
    """Bundle `nodes` into an assembly and imprint it with metadata

    Containerisation enables a tracking of version, author and origin
    for loaded assets.

    In Houdini it is not possible to next goemetry nodes in geometry nodes
    directly. To counter this we place a Object Network node calles ROOT
    in the HOUDINI_CONTAINERS node.

    Arguments:
        name (str): Name of resulting assembly
        namespace (str): Namespace under which to host container
        nodes (list): Long names of nodes to containerise
        context (dict): Asset information
        loader (str, optional): Name of loader used to produce this container.
        suffix (str, optional): Suffix of container, defaults to `_CON`.

    Returns:
        container (str): Name of container assembly

    """

    # Ensure AVALON_CONTAINERS subnet exists
    subnet = hou.node(AVALON_CONTAINERS)
    if subnet is None:
        obj_network = hou.node("/obj")
        subnet = obj_network.createNode("subnet",
                                        node_name="AVALON_CONTAINERS")

    # Create proper container name
    container_name = "{}_{}".format(name, suffix or "CON")
    container = hou.node("/obj/{}".format(name))
    container.setName(container_name)

    data = {
        "schema": "avalon-core:container-2.0",
        "id": "pyblish.avalon.container",
        "name": name,
        "namespace": namespace,
        "loader": str(loader),
        "representation": str(context["representation"]["_id"]),
    }

    lib.imprint(container, data)

    # "Parent" the container under the container network
    hou.moveNodesTo([container], subnet)

    subnet.node(container_name).moveToGoodPosition()

    return container

def parse_container(container, validate=True):
    """Return the container node's full container data.

    Args:
        container (hou.Node): A container node name.
        validate(bool): turn the validation for the container on or off

    Returns:
        dict: The container schema data for this container node.

    """
    data = lib.read(container)

    # Backwards compatibility pre-schemas for containers
    data["schema"] = data.get("schema", "avalon-core:container-1.0")

    # Append transient data
    data["objectName"] = container.path()
    data["node"] = container

    if validate:
        schema.validate(data)

    return data

def ls():
    containers = []
    for identifier in ("pyblish.avalon.container",
                       "pyblish.mindbender.container"):
        containers += lib.lsattr("id", identifier)

    for container in sorted(containers):
        data = parse_container(container)
        yield data

class Creator(api.Creator):
    """Creator plugin to create instances in Houdini

    To support the wide range of node types for render output (Alembic, VDB,
    Mantra) the Creator needs a node type to create the correct instance

    By default, if none is given, is `geometry`. An example of accepted node
    types: geometry, alembic, ifd (mantra)

    Please check the Houdini documentation for more node types.

    Tip: to find the exact node type to create press the `i` left of the node
    when hovering over a node. The information is visible under the name of
    the node.

    """

    def __init__(self, *args, **kwargs):
        api.Creator.__init__(self, *args, **kwargs)
        self.nodes = list()

    def process(self):
        """This is the base functionality to create instances in Houdini

        The selected nodes are stored in self to be used in an override method.
        This is currently necessary in order to support the multiple output
        types in Houdini which can only be rendered through their own node.

        Default node type if none is given is `geometry`

        It also makes it easier to apply custom settings per instance type

        Example of override method for Alembic:

            def process(self):
                instance =  super(CreateEpicNode, self, process()
                # Set paramaters for Alembic node
                instance.setParms(
                    {"sop_path": "$HIP/%s.abc" % self.nodes[0]}
                )

        Returns:
            hou.Node

        """

        if (self.options or {}).get("useSelection"):
            self.nodes = hou.selectedNodes()

        # Get the node type and remove it from the data, not needed
        node_type = self.data.pop("node_type", None)
        if node_type is None:
            node_type = "geometry"

        # Get out node
        out = hou.node("out")
        instance = out.createNode(node_type, node_name=self.name)
        instance.moveToGoodPosition()

        lib.imprint(instance, self.data)

        return instance

def on_file_event_callback(event):
    if event == hou.hipFileEventType.AfterLoad:
        api.emit("open", [event])
    elif event == hou.hipFileEventType.AfterSave:
        api.emit("save", [event])
    elif event == hou.hipFileEventType.BeforeSave:
        api.emit("before_save", [event])
    elif event == hou.hipFileEventType.AfterClear:
        api.emit("new", [event])

def on_houdini_initialize():

    main_window = hou.qt.mainWindow()
    self._parent = {main_window.objectName(): main_window}

def _register_callbacks():

    for handler, event in self._events.copy().items():
        if event is None:
            continue

        try:
            hou.hipFile.removeEventCallback(event)
        except RuntimeError as e:
            logger.info(e)

    self._events[on_file_event_callback] = hou.hipFile.addEventCallback(
        on_file_event_callback
    )
<commit_msg>Improve docstring, remove wrong documentation<commit_after># Standard library
import sys
import importlib

# Pyblish libraries
import pyblish.api

# Host libraries
import hou

# Local libraries
from . import lib
from ..lib import logger
from avalon import api, schema

self = sys.modules[__name__]
self._has_been_setup = False
self._parent = None
self._events = dict()

AVALON_CONTAINERS = "/obj/AVALON_CONTAINERS"
IS_HEADLESS = not hasattr(hou, "ui")

def install(config):
    """Setup integration
    Register plug-ins and integrate into the host

    """

    print("Registering callbacks")
    _register_callbacks()

    pyblish.api.register_host("houdini")
    pyblish.api.register_host("hython")
    pyblish.api.register_host("hpython")

    self._has_been_setup = True

    config = find_host_config(config)
    if hasattr(config, "install"):
        config.install()

def uninstall(config):
    """Uninstall Houdini-specific functionality of avalon-core.

    This function is called automatically on calling `api.uninstall()`.

    Args:
        config: configuration module

    """

    config = find_host_config(config)
    if hasattr(config, "uninstall"):
        config.uninstall()

    pyblish.api.deregister_host("hython")
    pyblish.api.deregister_host("hpython")
    pyblish.api.deregister_host("houdini")

def find_host_config(config):
    config_name = config.__name__
    try:
        config = importlib.import_module(config_name + ".houdini")
    except ImportError as exc:
        if str(exc) != "No module name {}".format(config_name + ".houdini"):
            raise
        config = None

    return config

def reload_pipeline(*args):
    """Attempt to reload pipeline at run-time.

    CAUTION: This is primarily for development and debugging purposes.

    """

    import importlib

    api.uninstall()

    for module in ("avalon.io",
                   "avalon.lib",
                   "avalon.pipeline",

                   "avalon.houdini.pipeline",
                   "avalon.houdini.lib",
                   "avalon.tools.loader.app",
                   "avalon.tools.creator.app",
                   "avalon.tools.manager.app",

                   # NOTE(marcus): These have circular depenendencies
                   #               that is preventing reloadability
                   # "avalon.tools.cbloader.delegates",
                   # "avalon.tools.cbloader.model",
                   # "avalon.tools.cbloader.widgets",
                   # "avalon.tools.cbloader.app",
                   # "avalon.tools.cbsceneinventory.model",
                   # "avalon.tools.cbsceneinventory.proxy",
                   # "avalon.tools.cbsceneinventory.app",
                   # "avalon.tools.projectmanager.dialogs",
                   # "avalon.tools.projectmanager.lib",
                   # "avalon.tools.projectmanager.model",
                   # "avalon.tools.projectmanager.style",
                   # "avalon.tools.projectmanager.widget",
                   # "avalon.tools.projectmanager.app",

                   "avalon.api",
                   "avalon.tools",
                   "avalon.houdini"):
        module = importlib.import_module(module)
        reload(module)

    self._parent = {hou.ui.mainQtWindow().objectName(): hou.ui.mainQtWindow()}

    import avalon.houdini
    api.install(avalon.houdini)

def _discover_gui():
    """Return the most desirable of the currently registered GUIs"""

    # Prefer last registered
    guis = reversed(pyblish.api.registered_guis())

    for gui in guis:
        try:
            gui = __import__(gui).show
        except (ImportError, AttributeError):
            continue
        else:
            return gui

def teardown():
    """Remove integration"""
    if not self._has_been_setup:
        return

    self._has_been_setup = False
    print("pyblish: Integration torn down successfully")

def containerise(name,
                 namespace,
                 nodes,
                 context,
                 loader=None,
                 suffix=""):
    """Bundle `nodes` into a subnet and imprint it with metadata

    Containerisation enables a tracking of version, author and origin
    for loaded assets.

    Arguments:
        name (str): Name of resulting assembly
        namespace (str): Namespace under which to host container
        nodes (list): Long names of nodes to containerise
        context (dict): Asset information
        loader (str, optional): Name of loader used to produce this container.
        suffix (str, optional): Suffix of container, defaults to `_CON`.

    Returns:
        container (str): Name of container assembly

    """

    # Ensure AVALON_CONTAINERS subnet exists
    subnet = hou.node(AVALON_CONTAINERS)
    if subnet is None:
        obj_network = hou.node("/obj")
        subnet = obj_network.createNode("subnet",
                                        node_name="AVALON_CONTAINERS")

    # Create proper container name
    container_name = "{}_{}".format(name, suffix or "CON")
    container = hou.node("/obj/{}".format(name))
    container.setName(container_name)

    data = {
        "schema": "avalon-core:container-2.0",
        "id": "pyblish.avalon.container",
        "name": name,
        "namespace": namespace,
        "loader": str(loader),
        "representation": str(context["representation"]["_id"]),
    }

    lib.imprint(container, data)

    # "Parent" the container under the container network
    hou.moveNodesTo([container], subnet)

    subnet.node(container_name).moveToGoodPosition()

    return container

def parse_container(container, validate=True):
    """Return the container node's full container data.

    Args:
        container (hou.Node): A container node name.
        validate(bool): turn the validation for the container on or off

    Returns:
        dict: The container schema data for this container node.

    """
    data = lib.read(container)

    # Backwards compatibility pre-schemas for containers
    data["schema"] = data.get("schema", "avalon-core:container-1.0")

    # Append transient data
    data["objectName"] = container.path()
    data["node"] = container

    if validate:
        schema.validate(data)

    return data

def ls():
    containers = []
    for identifier in ("pyblish.avalon.container",
                       "pyblish.mindbender.container"):
        containers += lib.lsattr("id", identifier)

    for container in sorted(containers):
        data = parse_container(container)
        yield data

class Creator(api.Creator):
    """Creator plugin to create instances in Houdini

    To support the wide range of node types for render output (Alembic, VDB,
    Mantra) the Creator needs a node type to create the correct instance

    By default, if none is given, is `geometry`. An example of accepted node
    types: geometry, alembic, ifd (mantra)

    Please check the Houdini documentation for more node types.

    Tip: to find the exact node type to create press the `i` left of the node
    when hovering over a node. The information is visible under the name of
    the node.

    """

    def __init__(self, *args, **kwargs):
        api.Creator.__init__(self, *args, **kwargs)
        self.nodes = list()

    def process(self):
        """This is the base functionality to create instances in Houdini

        The selected nodes are stored in self to be used in an override method.
        This is currently necessary in order to support the multiple output
        types in Houdini which can only be rendered through their own node.

        Default node type if none is given is `geometry`

        It also makes it easier to apply custom settings per instance type

        Example of override method for Alembic:

            def process(self):
                instance =  super(CreateEpicNode, self, process()
                # Set paramaters for Alembic node
                instance.setParms(
                    {"sop_path": "$HIP/%s.abc" % self.nodes[0]}
                )

        Returns:
            hou.Node

        """

        if (self.options or {}).get("useSelection"):
            self.nodes = hou.selectedNodes()

        # Get the node type and remove it from the data, not needed
        node_type = self.data.pop("node_type", None)
        if node_type is None:
            node_type = "geometry"

        # Get out node
        out = hou.node("out")
        instance = out.createNode(node_type, node_name=self.name)
        instance.moveToGoodPosition()

        lib.imprint(instance, self.data)

        return instance

def on_file_event_callback(event):
    if event == hou.hipFileEventType.AfterLoad:
        api.emit("open", [event])
    elif event == hou.hipFileEventType.AfterSave:
        api.emit("save", [event])
    elif event == hou.hipFileEventType.BeforeSave:
        api.emit("before_save", [event])
    elif event == hou.hipFileEventType.AfterClear:
        api.emit("new", [event])

def on_houdini_initialize():

    main_window = hou.qt.mainWindow()
    self._parent = {main_window.objectName(): main_window}

def _register_callbacks():

    for handler, event in self._events.copy().items():
        if event is None:
            continue

        try:
            hou.hipFile.removeEventCallback(event)
        except RuntimeError as e:
            logger.info(e)

    self._events[on_file_event_callback] = hou.hipFile.addEventCallback(
        on_file_event_callback
    )
<|endoftext|>


<commit_before>from urllib.error import HTTPError
from urllib.request import urlopen
from .weather import *

class Report:
    def __init__(self, raw):
        if " RMK " in raw:
            body, remarks = raw.split(" RMK ", 1)
        else:
            body, remarks = raw, None
        self.body = self._parse_body(body)
        self.remarks = self._parse_remarks(remarks)

    def __str__(self):
        raw = " ".join((str(part) for part in self.body))
        if self.remarks:
            raw += " RMK " + " ".join((str(part) for part in self.remarks))
        return raw

    def _parse_body(self, text):
        parts = text.split()
        if len(parts) == 0:
            raise ReportDecodeException

        if parts[0] in ("METAR", "SPECI"):
            self.type = parts[0]
            parts = parts[1:]
        else:
            self.type = None

        if "AUTO" in parts:
            self.modifier = "AUTO"
        # elif "COR" in parts:
        #     self.modifier = "COR"
        else:
            self.modifier = None
        if self.modifier:
            parts.remove(self.modifier)

        self.station = Station(parts[0])
        self.time = Time(parts[1])

        try:
            self.wind = Wind(text)
        except (WindDecodeException, IndexError):
            self.wind = None
            parts = parts[2:]
        else:
            parts = " ".join(parts[2:]).split(str(self.wind), 1)[1].split()  # remove used parts

        try:
            self.visibility = Visibility(" ".join(parts[:2]))
        except (VisibilityDecodeException, IndexError):
            self.visibility = None
        else:
            parts = text.split(str(self.visibility), 1)[1].split()  # remove used parts

        t = list()
        i = 0
        try:
            while True:
                t.append(RunwayVisualRange(parts[i]))
                i += 1
        except (RunwayVisualRangeDecodeException, IndexError):
            parts = parts[i:]
        self.runway_visual_range = tuple(t)

        t = list()
        i = 0
        try:
            while True:
                t.append(WeatherGroup(parts[i]))
                i += 1
        except (WeatherGroupDecodeException, IndexError):
            parts = parts[i:]
        self.weather_groups = tuple(t)

        t = list()
        i = 0
        try:
            while True:
                t.append(SkyCondition(parts[i]))
                i += 1
        except (SkyConditionDecodeException, IndexError):
            parts = parts[i:]
        self.sky_conditions = tuple(t)

        try:
            self.temperature = Temperature(parts[0])
        except (TemperatureDecodeException, IndexError):
            self.temperature = None
        else:
            parts = parts[1:]

        try:
            self.altimeter_setting = Pressure(parts[0])
        except (PressureDecodeException, IndexError):
            self.altimeter_setting = None

        body = [self.type, self.station, self.time, self.modifier, self.wind, self.visibility,
                *self.runway_visual_range, *self.weather_groups, *self.sky_conditions, self.temperature,
                self.altimeter_setting, " ".join(parts[1:])]
        return list(filter(None, body))

    def _parse_remarks(self, text):
        if not text:
            return []
        else:
            return [Remarks(text)]  # TODO: consider more detail from remarks?

    @staticmethod
    def retrieve(code):
        url = "http://weather.noaa.gov/pub/data/observations/metar/stations/%s.TXT"
        try:
            with urlopen(url % code.upper()) as response:
                metar = response.read().decode('utf-8').splitlines()[1].strip()
                return Report(metar)
        except HTTPError:
            return None
<commit_msg>unnecessary parenthesis<commit_after>from urllib.error import HTTPError
from urllib.request import urlopen
from .weather import *

class Report:
    def __init__(self, raw):
        if " RMK " in raw:
            body, remarks = raw.split(" RMK ", 1)
        else:
            body, remarks = raw, None
        self.body = self._parse_body(body)
        self.remarks = self._parse_remarks(remarks)

    def __str__(self):
        raw = " ".join((str(part) for part in self.body))
        if self.remarks:
            raw += " RMK " + " ".join(str(part) for part in self.remarks)
        return raw

    def _parse_body(self, text):
        parts = text.split()
        if len(parts) == 0:
            raise ReportDecodeException

        if parts[0] in ("METAR", "SPECI"):
            self.type = parts[0]
            parts = parts[1:]
        else:
            self.type = None

        if "AUTO" in parts:
            self.modifier = "AUTO"
        # elif "COR" in parts:
        #     self.modifier = "COR"
        else:
            self.modifier = None
        if self.modifier:
            parts.remove(self.modifier)

        self.station = Station(parts[0])
        self.time = Time(parts[1])

        try:
            self.wind = Wind(text)
        except (WindDecodeException, IndexError):
            self.wind = None
            parts = parts[2:]
        else:
            parts = " ".join(parts[2:]).split(str(self.wind), 1)[1].split()  # remove used parts

        try:
            self.visibility = Visibility(" ".join(parts[:2]))
        except (VisibilityDecodeException, IndexError):
            self.visibility = None
        else:
            parts = text.split(str(self.visibility), 1)[1].split()  # remove used parts

        t = list()
        i = 0
        try:
            while True:
                t.append(RunwayVisualRange(parts[i]))
                i += 1
        except (RunwayVisualRangeDecodeException, IndexError):
            parts = parts[i:]
        self.runway_visual_range = tuple(t)

        t = list()
        i = 0
        try:
            while True:
                t.append(WeatherGroup(parts[i]))
                i += 1
        except (WeatherGroupDecodeException, IndexError):
            parts = parts[i:]
        self.weather_groups = tuple(t)

        t = list()
        i = 0
        try:
            while True:
                t.append(SkyCondition(parts[i]))
                i += 1
        except (SkyConditionDecodeException, IndexError):
            parts = parts[i:]
        self.sky_conditions = tuple(t)

        try:
            self.temperature = Temperature(parts[0])
        except (TemperatureDecodeException, IndexError):
            self.temperature = None
        else:
            parts = parts[1:]

        try:
            self.altimeter_setting = Pressure(parts[0])
        except (PressureDecodeException, IndexError):
            self.altimeter_setting = None

        body = [self.type, self.station, self.time, self.modifier, self.wind, self.visibility,
                *self.runway_visual_range, *self.weather_groups, *self.sky_conditions, self.temperature,
                self.altimeter_setting, " ".join(parts[1:])]
        return list(filter(None, body))

    def _parse_remarks(self, text):
        if not text:
            return []
        else:
            return [Remarks(text)]  # TODO: consider more detail from remarks?

    @staticmethod
    def retrieve(code):
        url = "http://weather.noaa.gov/pub/data/observations/metar/stations/%s.TXT"
        try:
            with urlopen(url % code.upper()) as response:
                metar = response.read().decode('utf-8').splitlines()[1].strip()
                return Report(metar)
        except HTTPError:
            return None
<|endoftext|>