<commit_before>"""
Directory, which is for utility/non-metric endpoints.
"""

import datetime
import sqlalchemy as s
import pandas as pd
import base64
from augur.util import logger, annotate, add_metrics

@annotate(tag='repo-groups')
def repo_groups(self):
    """
    Returns number of lines changed per author per day

    :param repo_url: the repository's URL
    """
    repoGroupsSQL = s.sql.text("""
        SELECT *
        FROM repo_groups
        ORDER BY rg_name
    """)
    results = pd.read_sql(repoGroupsSQL, self.database)
    return results

@annotate(tag='downloaded-repos')
def downloaded_repos(self):
    """
    Returns all repository names, URLs, and base64 URLs in the facade database
    """
    downloadedReposSQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time ,
            rg_name,
            repo.repo_group_id
        FROM
            repo
            left outer join
            (select repo_id,    COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( * ) as issues_all_time from issues where issues.pull_request IS NULL  group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        order by rg_name
    """)
    results = pd.read_sql(downloadedReposSQL, self.database)
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    # if self.projects:
    #     results = results[results.project_name.isin(self.projects)]
    if self.projects:
          results = results[results.project_name.isin(self.projects)]

    b64_urls = []
    for i in results.index:
        b64_urls.append(base64.b64encode((results.at[i, 'url']).encode()))
    results['base64_url'] = b64_urls

    return results

@annotate(tag='repos-in-repo-groups')
def repos_in_repo_groups(self, repo_group_id):
    """
    Returns a list of all the repos in a repo_group

    :param repo_group_id: The repository's repo_group_id
    """
    repos_in_repo_groups_SQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time
        FROM
            repo
            left outer join
            (select repo_id, COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE
            repo_groups.repo_group_id = :repo_group_id
        ORDER BY commits_all_time DESC
    """)

    results = pd.read_sql(repos_in_repo_groups_SQL, self.database, params={'repo_group_id': repo_group_id})
    return results

@annotate(tag='get-repo-by-git-name')
def get_repo_by_git_name(self, owner, repo):
    """
    Returns repo id and repo group id by owner and repo

    :param owner: the owner of the repo
    :param repo: the name of the repo
    """
    getRepoSQL = s.sql.text("""
        SELECT repo.repo_id, repo.repo_group_id, rg_name
        FROM repo JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE repo_name = :repo AND repo_path LIKE :owner
        GROUP BY repo_id, rg_name
    """)

    results = pd.read_sql(getRepoSQL, self.database, params={'owner': '%{}_'.format(owner), 'repo': repo,})

    return results

@annotate(tag='get-repo-by-name')
def get_repo_by_name(self, rg_name, repo_name):
    """
    Returns repo id and repo group id by rg_name and repo_name

    :param rg_name: the repo group of the repo
    :param repo_name: the name of the repo
    """

    repoSQL = s.sql.text("""
        SELECT repo_id, repo.repo_group_id, repo_git as url
        FROM repo, repo_groups
        WHERE repo.repo_group_id = repo_groups.repo_group_id
        AND LOWER(rg_name) = LOWER(:rg_name)
        AND LOWER(repo_name) = LOWER(:repo_name)
    """)
    results = pd.read_sql(repoSQL, self.database, params={'rg_name': rg_name, 'repo_name': repo_name})
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    return results

def get_group_by_name(self, rg_name):
    """
    Returns repo group id by repo group name

    :param rg_name:
    """
    groupSQL = s.sql.text("""
        SELECT repo_group_id, rg_name
        FROM repo_groups
        WHERE lower(rg_name) = lower(:rg_name)
    """)
    results = pd.read_sql(groupSQL, self.database, params={'rg_name': rg_name})
    return results

@annotate(tag='dosocs-repos')
def get_repos_for_dosocs(self):
    """ Returns a list of repos along with their repo_id & path """
    get_repos_for_dosocs_SQL = s.sql.text("""
        SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) AS path
        FROM settings a, repo b
        WHERE a.setting='repo_directory'
    """)

    results = pd.read_sql(get_repos_for_dosocs_SQL, self.database)
    return results

@annotate(tag="get-issues")
def get_issues(self, repo_group_id, repo_id=None):
    if not repo_id:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY
            FROM issues,
                issue_events
            WHERE issues.repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id)
            AND issues.issue_id = issue_events.issue_id
            AND issues.pull_request is NULL
            GROUP BY issues.issue_id
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_group_id': repo_group_id})
        return results
    else:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY,
                repo_name
            FROM issues JOIN repo ON issues.repo_id = repo.repo_id, issue_events
            WHERE issues.repo_id = :repo_id
            AND issues.pull_request IS NULL
            AND issues.issue_id = issue_events.issue_id
            GROUP BY issues.issue_id, repo_name
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_id': repo_id})
        return results

@annotate(tag="aggregate-summary")
def aggregate_summary(self, repo_group_id, repo_id=None, begin_date=None, end_date=None):

    if not begin_date:
        begin_date = datetime.datetime.now()
        # Subtract 1 year and leap year check
        try:
            begin_date = begin_date.replace(year=begin_date.year-1)
        except ValueError:
            begin_date = begin_date.replace(year=begin_date.year-1, day=begin_date.day-1)
        begin_date = begin_date.strftime('%Y-%m-%d')
    if not end_date:
        end_date = datetime.datetime.now().strftime('%Y-%m-%d')

    if not repo_id:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id JOIN repo ON issues.repo_id = repo.repo_id
                    WHERE action = 'merged'
                    AND repo_group_id = :repo_group_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits JOIN repo ON commits.repo_id = repo.repo_id
                    WHERE repo_group_id = :repo_group_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_group_id': repo_group_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results
    else:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id
                    WHERE action = 'merged'
                    AND repo_id = :repo_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits
                    WHERE repo_id = :repo_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_id': repo_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results

def create_util_metrics(metrics):
    add_metrics(metrics, __name__)
<commit_msg>One more API returning alphabetical results.<commit_after>"""
Directory, which is for utility/non-metric endpoints.
"""

import datetime
import sqlalchemy as s
import pandas as pd
import base64
from augur.util import logger, annotate, add_metrics

@annotate(tag='repo-groups')
def repo_groups(self):
    """
    Returns number of lines changed per author per day

    :param repo_url: the repository's URL
    """
    repoGroupsSQL = s.sql.text("""
        SELECT *
        FROM repo_groups
        ORDER BY rg_name
    """)
    results = pd.read_sql(repoGroupsSQL, self.database)
    return results

@annotate(tag='downloaded-repos')
def downloaded_repos(self):
    """
    Returns all repository names, URLs, and base64 URLs in the facade database
    """
    downloadedReposSQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time ,
            rg_name,
            repo.repo_group_id
        FROM
            repo
            left outer join
            (select repo_id,    COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( * ) as issues_all_time from issues where issues.pull_request IS NULL  group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        order by rg_name
    """)
    results = pd.read_sql(downloadedReposSQL, self.database)
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    # if self.projects:
    #     results = results[results.project_name.isin(self.projects)]
    if self.projects:
          results = results[results.project_name.isin(self.projects)]

    b64_urls = []
    for i in results.index:
        b64_urls.append(base64.b64encode((results.at[i, 'url']).encode()))
    results['base64_url'] = b64_urls

    return results

@annotate(tag='repos-in-repo-groups')
def repos_in_repo_groups(self, repo_group_id):
    """
    Returns a list of all the repos in a repo_group

    :param repo_group_id: The repository's repo_group_id
    """
    repos_in_repo_groups_SQL = s.sql.text("""
        SELECT
            repo.repo_id,
            repo.repo_name,
            repo.description,
            repo.repo_git AS url,
            repo.repo_status,
            a.commits_all_time,
            b.issues_all_time
        FROM
            repo
            left outer join
            (select repo_id, COUNT ( commits.cmt_id ) AS commits_all_time from commits group by repo_id ) a on
            repo.repo_id = a.repo_id
            left outer join
            (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
            on
            repo.repo_id = b.repo_id
            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE
            repo_groups.repo_group_id = :repo_group_id
        ORDER BY repo.repo_name DESC
    """)

    results = pd.read_sql(repos_in_repo_groups_SQL, self.database, params={'repo_group_id': repo_group_id})
    return results

@annotate(tag='get-repo-by-git-name')
def get_repo_by_git_name(self, owner, repo):
    """
    Returns repo id and repo group id by owner and repo

    :param owner: the owner of the repo
    :param repo: the name of the repo
    """
    getRepoSQL = s.sql.text("""
        SELECT repo.repo_id, repo.repo_group_id, rg_name
        FROM repo JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
        WHERE repo_name = :repo AND repo_path LIKE :owner
        GROUP BY repo_id, rg_name
    """)

    results = pd.read_sql(getRepoSQL, self.database, params={'owner': '%{}_'.format(owner), 'repo': repo,})

    return results

@annotate(tag='get-repo-by-name')
def get_repo_by_name(self, rg_name, repo_name):
    """
    Returns repo id and repo group id by rg_name and repo_name

    :param rg_name: the repo group of the repo
    :param repo_name: the name of the repo
    """

    repoSQL = s.sql.text("""
        SELECT repo_id, repo.repo_group_id, repo_git as url
        FROM repo, repo_groups
        WHERE repo.repo_group_id = repo_groups.repo_group_id
        AND LOWER(rg_name) = LOWER(:rg_name)
        AND LOWER(repo_name) = LOWER(:repo_name)
    """)
    results = pd.read_sql(repoSQL, self.database, params={'rg_name': rg_name, 'repo_name': repo_name})
    results['url'] = results['url'].apply(lambda datum: datum.split('//')[1])
    return results

def get_group_by_name(self, rg_name):
    """
    Returns repo group id by repo group name

    :param rg_name:
    """
    groupSQL = s.sql.text("""
        SELECT repo_group_id, rg_name
        FROM repo_groups
        WHERE lower(rg_name) = lower(:rg_name)
    """)
    results = pd.read_sql(groupSQL, self.database, params={'rg_name': rg_name})
    return results

@annotate(tag='dosocs-repos')
def get_repos_for_dosocs(self):
    """ Returns a list of repos along with their repo_id & path """
    get_repos_for_dosocs_SQL = s.sql.text("""
        SELECT b.repo_id, CONCAT(a.value || b.repo_group_id || chr(47) || b.repo_path || b.repo_name) AS path
        FROM settings a, repo b
        WHERE a.setting='repo_directory'
    """)

    results = pd.read_sql(get_repos_for_dosocs_SQL, self.database)
    return results

@annotate(tag="get-issues")
def get_issues(self, repo_group_id, repo_id=None):
    if not repo_id:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY
            FROM issues,
                issue_events
            WHERE issues.repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id)
            AND issues.issue_id = issue_events.issue_id
            AND issues.pull_request is NULL
            GROUP BY issues.issue_id
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_group_id': repo_group_id})
        return results
    else:
        issuesSQL = s.sql.text("""
            SELECT issue_title,
                issues.issue_id,
                issues.repo_id,
                issues.html_url,
                issue_state                                 AS STATUS,
                issues.created_at                           AS DATE,
                count(issue_events.event_id),
                MAX(issue_events.created_at)                AS LAST_EVENT_DATE,
                EXTRACT(DAY FROM NOW() - issues.created_at) AS OPEN_DAY,
                repo_name
            FROM issues JOIN repo ON issues.repo_id = repo.repo_id, issue_events
            WHERE issues.repo_id = :repo_id
            AND issues.pull_request IS NULL
            AND issues.issue_id = issue_events.issue_id
            GROUP BY issues.issue_id, repo_name
            ORDER by OPEN_DAY DESC
        """)
        results = pd.read_sql(issuesSQL, self.database, params={'repo_id': repo_id})
        return results

@annotate(tag="aggregate-summary")
def aggregate_summary(self, repo_group_id, repo_id=None, begin_date=None, end_date=None):

    if not begin_date:
        begin_date = datetime.datetime.now()
        # Subtract 1 year and leap year check
        try:
            begin_date = begin_date.replace(year=begin_date.year-1)
        except ValueError:
            begin_date = begin_date.replace(year=begin_date.year-1, day=begin_date.day-1)
        begin_date = begin_date.strftime('%Y-%m-%d')
    if not end_date:
        end_date = datetime.datetime.now().strftime('%Y-%m-%d')

    if not repo_id:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info JOIN repo ON repo_info.repo_id = repo.repo_id
                WHERE repo_group_id = :repo_group_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id JOIN repo ON issues.repo_id = repo.repo_id
                    WHERE action = 'merged'
                    AND repo_group_id = :repo_group_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits JOIN repo ON commits.repo_id = repo.repo_id
                    WHERE repo_group_id = :repo_group_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_group_id': repo_group_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results
    else:
        summarySQL = s.sql.text("""
            SELECT
            (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT watchers_count AS watcher_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS watcher_count,
            (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT stars_count AS stars_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS stars_count,
            (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                ORDER BY last_updated DESC
                LIMIT 1
            ) - (
                SELECT fork_count AS fork_count
                FROM repo_info
                WHERE repo_id = :repo_id
                AND last_updated >= date_trunc('day', NOW() - INTERVAL '1 year')
                ORDER BY last_updated ASC
                LIMIT 1
            ) AS fork_count,
            (
                SELECT count(*) AS merged_count
                FROM (
                    SELECT DISTINCT issue_events.issue_id
                    FROM issue_events JOIN issues ON issues.issue_id = issue_events.issue_id
                    WHERE action = 'merged'
                    AND repo_id = :repo_id
                    AND issue_events.created_at BETWEEN :begin_date AND :end_date
                ) a
            ) AS merged_count,
            committer_count, commit_count FROM (
                SELECT count(cmt_author_name) AS committer_count, sum(commit_count) AS commit_count
                FROM (
                    SELECT DISTINCT cmt_author_name, COUNT(cmt_id) AS commit_count FROM commits
                    WHERE repo_id = :repo_id
                    AND commits.cmt_committer_date BETWEEN :begin_date AND :end_date
                    GROUP BY cmt_author_name
                ) temp
            ) commit_data
        """)
        results = pd.read_sql(summarySQL, self.database, params={'repo_id': repo_id,
                                                        'begin_date': begin_date, 'end_date': end_date})
        return results

def create_util_metrics(metrics):
    add_metrics(metrics, __name__)
<|endoftext|>


<commit_before>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    return inner_problem_check(problems(reversed(pending), done), done, verbosity)

def backwards_problems(pending, done, verbosity):
    return inner_problem_check(problems(pending, done), done, verbosity)

def inner_problem_check(problems, done, verbosity):
    "Takes a set of possible problems and gets the actual issues out of it."
    result = []
    for last, migration in problems:
        # 'Last' is the last applied migration. Step back from it until we
        # either find nothing wrong, or we find something.
        to_check = list(last.dependencies)
        while to_check:
            checking = to_check.pop()
            if checking not in done:
                # That's bad. Error.
                if verbosity:
                    print (" ! Migration %s should not have been applied "
                           "before %s but was." % (last, checking))
                result.append((last, checking))
            else:
                to_check.extend(checking.dependencies)
    return result

def check_migration_histories(histories, delete_ghosts=False, ignore_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        else:
            exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        elif not ignore_ghosts:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity, interactive):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity, interactive=interactive)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity, interactive=interactive)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False, ignore_ghosts=False, interactive=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied = MigrationHistory.objects.filter(applied__isnull=False)
    # If we're using a different database, use that
    if database != DEFAULT_DB_ALIAS:
        applied = applied.using(database)
        south.db.db = south.db.dbs[database]
        # We now have to make sure the migrations are all reloaded, as they'll
        # have imported the old value of south.db.db.
        Migrations.invalidate_all_modules()

    south.db.db.debug = (verbosity > 1)
    applied = check_migration_histories(applied, delete_ghosts, ignore_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied, migrations,
                                                  verbosity, interactive)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan, database)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    else:
        if verbosity:
            # Say there's nothing.
            print '- Nothing to migrate.'
        # If we have initial data enabled, and we're at the most recent
        # migration, do initial data.
        # Note: We use a fake Forwards() migrator here. It's never used really.
        if load_initial_data:
            migrator = LoadInitialDataMigrator(migrator=Forwards(verbosity=verbosity))
            migrator.load_initial_data(target, db=database)
        # Send signal.
        post_migrate.send(None, app=app_label)
<commit_msg>Add a "current-1" virtual migration name. E.g.: manage.py migrate dummyapp current-1<commit_after>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    return inner_problem_check(problems(reversed(pending), done), done, verbosity)

def backwards_problems(pending, done, verbosity):
    return inner_problem_check(problems(pending, done), done, verbosity)

def inner_problem_check(problems, done, verbosity):
    "Takes a set of possible problems and gets the actual issues out of it."
    result = []
    for last, migration in problems:
        # 'Last' is the last applied migration. Step back from it until we
        # either find nothing wrong, or we find something.
        to_check = list(last.dependencies)
        while to_check:
            checking = to_check.pop()
            if checking not in done:
                # That's bad. Error.
                if verbosity:
                    print (" ! Migration %s should not have been applied "
                           "before %s but was." % (last, checking))
                result.append((last, checking))
            else:
                to_check.extend(checking.dependencies)
    return result

def check_migration_histories(histories, delete_ghosts=False, ignore_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        else:
            exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        elif not ignore_ghosts:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity, interactive):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity, interactive=interactive)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity, interactive=interactive)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False, ignore_ghosts=False, interactive=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied = MigrationHistory.objects.filter(app_name=app_label, applied__isnull=False).order_by('applied')
    # If we're using a different database, use that
    if database != DEFAULT_DB_ALIAS:
        applied = applied.using(database)
        south.db.db = south.db.dbs[database]
        # We now have to make sure the migrations are all reloaded, as they'll
        # have imported the old value of south.db.db.
        Migrations.invalidate_all_modules()

    south.db.db.debug = (verbosity > 1)

    if target_name == 'current-1':
        if applied.count() > 1:
            previous_migration = applied[applied.count() - 2]
            if verbosity:
                print 'previous_migration: %s (applied: %s)' % (previous_migration.migration, previous_migration.applied)
            target_name = previous_migration.migration
        else:
            if verbosity:
                print 'previous_migration: zero'
            target_name = 'zero'

    applied = check_migration_histories(applied, delete_ghosts, ignore_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied, migrations,
                                                  verbosity, interactive)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan, database)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    else:
        if verbosity:
            # Say there's nothing.
            print '- Nothing to migrate.'
        # If we have initial data enabled, and we're at the most recent
        # migration, do initial data.
        # Note: We use a fake Forwards() migrator here. It's never used really.
        if load_initial_data:
            migrator = LoadInitialDataMigrator(migrator=Forwards(verbosity=verbosity))
            migrator.load_initial_data(target, db=database)
        # Send signal.
        post_migrate.send(None, app=app_label)
<|endoftext|>


<commit_before>###############################################################################
#
#   apogee.tools.download: download APOGEE data files
#
###############################################################################
import os
import sys
import shutil
import tempfile
import subprocess
import numpy
from apogee.tools import path
_DR10_URL= 'http://data.sdss3.org/sas/dr10'
_DR12_URL= 'http://data.sdss3.org/sas/dr12'
_PROPRIETARY_URL= 'http://data.sdss.org/sas/bosswork'
_MAX_NTRIES= 2
_ERASESTR= "                                                                                "
def allStar(dr=None):
    """
    NAME:
       allStar
    PURPOSE:
       download the allStar file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.allStarPath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path, hacked from aspcapStar path
    aspPath= path.aspcapStarPath(4140,'dum',dr=dr)
    downloadPath= aspPath.replace(os.path.join(path._APOGEE_DATA,
                                               'dr%s' % dr),
                                  _base_url(dr=dr))
    head, tail= os.path.split(downloadPath) #strips off filename
    downloadPath, tail= os.path.split(head) #strips off location_id
    downloadPath= os.path.join(downloadPath,os.path.basename(filePath))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def allVisit(dr=None):
    """
    NAME:
       allVisit
    PURPOSE:
       download the allVisit file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.allVisitPath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path, hacked from aspcapStar path
    aspPath= path.aspcapStarPath(4140,'dum',dr=dr)
    downloadPath= aspPath.replace(os.path.join(path._APOGEE_DATA,
                                               'dr%s' % dr),
                                  _base_url(dr=dr))
    head, tail= os.path.split(downloadPath) #strips off filename
    downloadPath, tail= os.path.split(head) #strips off location_id
    downloadPath= os.path.join(downloadPath,os.path.basename(filePath))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def rcsample(dr=None):
    """
    NAME:
       rcsample
    PURPOSE:
       download the rcsample file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.rcsamplePath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath=\
        os.path.join(_base_url(dr=dr),
                     'apogee/vac/apogee-rc/cat/apogee-rc-DR%s.fits' % dr)
    _download_file(downloadPath,filePath,dr,verbose=False)
    return None

def aspcapStar(loc_id,apogee_id,dr=None):
    """
    NAME:
       aspcapStar
    PURPOSE:
       download an aspcapStar file
    INPUT:
       loc_id - location ID
       apogee_id - APOGEE ID of the star
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-25 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.aspcapStarPath(loc_id,apogee_id,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                'dr%s' % dr),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr)
    return None

def apStar(loc_id,apogee_id,dr=None):
    """
    NAME:
       apStar
    PURPOSE:
       download an apStar file
    INPUT:
       loc_id - location ID
       apogee_id - APOGEE ID of the star
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-01-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.apStarPath(loc_id,apogee_id,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                'dr%s' % dr),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr)
    return None

def modelSpec(lib='GK',teff=4500,logg=2.5,metals=0.,
              cfe=0.,nfe=0.,afe=0.,vmicro=2.,
              dr=None,rmHDU1=True,rmHDU2=True):
    """
    NAME:
       modelSpec
    PURPOSE:
       download a model spectrum file
    INPUT:
       lib= ('GK') spectral library
       teff= (4500) grid-point Teff
       logg= (2.5) grid-point logg
       metals= (0.) grid-point metallicity
       cfe= (0.) grid-point carbon-enhancement
       nfe= (0.) grid-point nitrogen-enhancement
       afe= (0.) grid-point alpha-enhancement
       vmicro= (2.) grid-point microturbulence
       dr= return the path corresponding to this data release
       rmHUD1= (True) if True, rm the first (v. large) HDU with the high-resolution model spectrum
       rmHDU2= (True) if True, rm the second (quite large) HDU with the model spectrum convolved with the LSF
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-01-20 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.modelSpecPath(lib=lib,teff=teff,logg=logg,metals=metals,
                                 cfe=cfe,nfe=nfe,afe=afe,vmicro=vmicro,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                'dr%s' % dr),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    # Post-processing of the file, removing the big first HDU or the first two for local storage
    if rmHDU1 or rmHDU2:
        # Open the file, need to use astropy's fits reader, bc the file has issues
        import astropy.io.fits as apyfits
        from astropy.utils.exceptions import AstropyUserWarning
        import warnings
        warnings.filterwarnings('ignore',category=AstropyUserWarning)
        hdulist= apyfits.open(filePath)
        if rmHDU1:
            hdu= apyfits.PrimaryHDU(numpy.zeros((2,2)))
        else:
            hdu= hdulist[0].copy()
        inp= [hdu]
        if rmHDU2:
            hdu2= apyfits.ImageHDU(numpy.zeros((2,2)))
        else:
            hdu2= hdulist[1].copy()
        inp.append(hdu2)
        inp.extend(hdulist[2:])
        newHdulist= apyfits.HDUList(inp)
        # Fix any issues in the headers
        for ii in range(5):
            if '[A/M]' in newHdulist[ii].header:
                newHdulist[ii].header['AM']= newHdulist[ii].header.pop('[A/M]',None)
            if '[C/M]' in newHdulist[ii].header:
                newHdulist[ii].header['CM']= newHdulist[ii].header.pop('[C/M]',None)
            if '[N/M]' in newHdulist[ii].header:
                newHdulist[ii].header['NM']= newHdulist[ii].header.pop('[N/M]',None)
        # Overwrite file
        newHdulist.writeto(filePath,clobber=True,output_verify='silentfix')
    return None

def ferreModelLibrary(lib='GK',pca=True,sixd=True,unf=False,dr=None,
                      convertToBin=True):
    """
    NAME:
       ferreModelLibrary
    PURPOSE:
       download a FERRE model library
    INPUT:
       lib= ('GK') spectral library
       dr= return the path corresponding to this data release
       pca= (True) if True, download the PCA compressed library
       sixd= (True) if True, download the 6D library (w/o vmicro)
       unf= (False) if True, download the binary library (otherwise ascii)
       convertToBin= (True) if True and not unf, convert the ascii file to binary using ferre's ascii2bin (which has to be on the path)
    OUTPUT:
       (none; just downloads; also downloads the corresponding .hdr)
    HISTORY:
       2015-01-21 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.ferreModelLibraryPath(lib=lib,dr=dr,pca=pca,
                                         sixd=sixd,unf=unf)
    if not os.path.exists(filePath):
        # Create the file path
        downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                    _dr_string(dr)),
                                       _base_url(dr=dr))
        _download_file(downloadPath,filePath,dr,verbose=True)
        if convertToBin:
            sys.stdout.write('\r'+"Converting ascii model library to binary (can take a few minutes) ...\r")
            sys.stdout.flush()
            try:
                p= subprocess.Popen(['ascii2bin'],stdout=subprocess.PIPE,
                                    stdin=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=os.path.dirname(filePath))
                p.stdin.write(os.path.basename(filePath)+'\n')
                p.stdin.write('unf\n')
                stdout, stderr= p.communicate()
            except subprocess.CalledProcessError:
                print("Conversion of %s to binary failed ..." % (os.path.basename(filePath)))
            sys.stdout.write('\r'+_ERASESTR+'\r')
            sys.stdout.flush()
    # Also download the header
    if unf:
        headerFilePath= filePath.replace('.unf','.hdr')
    else:
        headerFilePath= filePath.replace('.dat','.hdr')
    if os.path.exists(headerFilePath): return None
    headerDownloadPath= headerFilePath.replace(os.path.join(path._APOGEE_DATA,
                                                            'dr%s' % dr),
                                               _base_url(dr=dr))
    _download_file(headerDownloadPath,headerFilePath,dr,verbose=True)
    return None

def modelAtmosphere(lib='kurucz_filled',teff=4500,logg=2.5,metals=0.,
                    cfe=0.,nfe=0.,afe=0.,vmicro=2.,dr=None):
    """
    NAME:
       modelAtmosphere
    PURPOSE:
       download a model atmosphere
    INPUT:
       lib= ('kurucz_filled') model atmosphere library
       teff= (4500) grid-point Teff
       logg= (2.5) grid-point logg
       metals= (0.) grid-point metallicity
       cfe= (0.) grid-point carbon-enhancement
       afe= (0.) grid-point alpha-enhancement
       vmicro= (2.) grid-point microturbulence
       dr= return the path corresponding to this data release
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-02-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= 'X'
    # First make sure the file doesn't exist
    filePath= path.modelAtmospherePath(lib=lib,teff=teff,logg=logg,
                                       metals=metals,cfe=cfe,afe=afe,
                                       vmicro=2.,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                'dr%s' % dr),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def linelist(linelist,dr=None):
    """
    NAME:
       linelist
    PURPOSE:
       download a linelist
    INPUT:
       linelist - name of the linelist
       dr= return the path corresponding to this data release
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-02-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= 'X'
    # First make sure the file doesn't exist
    filePath= path.linelistPath(linelist,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                'dr%s' % dr),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def _download_file(downloadPath,filePath,dr,verbose=False):
    sys.stdout.write('\r'+"Downloading file %s ...\r" \
                         % (os.path.basename(filePath)))
    sys.stdout.flush()
    try:
        # make all intermediate directories
        os.makedirs(os.path.dirname(filePath))
    except OSError: pass
    # Safe way of downloading
    downloading= True
    interrupted= False
    file, tmp_savefilename= tempfile.mkstemp()
    os.close(file) #Easier this way
    ntries= 0
    while downloading:
        try:
            cmd= ['wget','%s' % downloadPath,
                  '-O','%s' % tmp_savefilename]
            if not verbose: cmd.append('-q')
            subprocess.check_call(cmd)
            shutil.move(tmp_savefilename,filePath)
            downloading= False
            if interrupted:
                raise KeyboardInterrupt
        except subprocess.CalledProcessError:
            if not downloading: #Assume KeyboardInterrupt
                raise
            elif ntries > _MAX_NTRIES:
                raise IOError('File %s does not appear to exist on the server ...' % (os.path.basename(filePath)))
            sys.stdout.write('\r'+"KeyboardInterrupt ignored while downloading ...\r")
            sys.stdout.flush()
            os.remove(tmp_savefilename)
            interrupted= True
            ntries+= 1
        finally:
            if os.path.exists(tmp_savefilename):
                os.remove(tmp_savefilename)
    sys.stdout.write('\r'+_ERASESTR+'\r')
    sys.stdout.flush()
    return None

def _base_url(dr,rc=False):
    if dr == '10': return _DR10_URL
    elif dr == '12': return _DR12_URL
    else: return _PROPRIETARY_URL

def _dr_string(dr):
    if dr == 'bosswork': return 'bosswork'
    else: return 'dr%s' % dr
<commit_msg>switch everything to using dr_string<commit_after>###############################################################################
#
#   apogee.tools.download: download APOGEE data files
#
###############################################################################
import os
import sys
import shutil
import tempfile
import subprocess
import numpy
from apogee.tools import path
_DR10_URL= 'http://data.sdss3.org/sas/dr10'
_DR12_URL= 'http://data.sdss3.org/sas/dr12'
_PROPRIETARY_URL= 'http://data.sdss.org/sas/bosswork'
_MAX_NTRIES= 2
_ERASESTR= "                                                                                "
def allStar(dr=None):
    """
    NAME:
       allStar
    PURPOSE:
       download the allStar file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.allStarPath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path, hacked from aspcapStar path
    aspPath= path.aspcapStarPath(4140,'dum',dr=dr)
    downloadPath= aspPath.replace(os.path.join(path._APOGEE_DATA,
                                               _dr_string(dr)),
                                  _base_url(dr=dr))
    head, tail= os.path.split(downloadPath) #strips off filename
    downloadPath, tail= os.path.split(head) #strips off location_id
    downloadPath= os.path.join(downloadPath,os.path.basename(filePath))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def allVisit(dr=None):
    """
    NAME:
       allVisit
    PURPOSE:
       download the allVisit file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.allVisitPath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path, hacked from aspcapStar path
    aspPath= path.aspcapStarPath(4140,'dum',dr=dr)
    downloadPath= aspPath.replace(os.path.join(path._APOGEE_DATA,
                                               _dr_string(dr)),
                                  _base_url(dr=dr))
    head, tail= os.path.split(downloadPath) #strips off filename
    downloadPath, tail= os.path.split(head) #strips off location_id
    downloadPath= os.path.join(downloadPath,os.path.basename(filePath))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def rcsample(dr=None):
    """
    NAME:
       rcsample
    PURPOSE:
       download the rcsample file
    INPUT:
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-26 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.rcsamplePath(dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath=\
        os.path.join(_base_url(dr=dr),
                     'apogee/vac/apogee-rc/cat/apogee-rc-DR%s.fits' % dr)
    _download_file(downloadPath,filePath,dr,verbose=False)
    return None

def aspcapStar(loc_id,apogee_id,dr=None):
    """
    NAME:
       aspcapStar
    PURPOSE:
       download an aspcapStar file
    INPUT:
       loc_id - location ID
       apogee_id - APOGEE ID of the star
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2014-11-25 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.aspcapStarPath(loc_id,apogee_id,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                _dr_string(dr)),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr)
    return None

def apStar(loc_id,apogee_id,dr=None):
    """
    NAME:
       apStar
    PURPOSE:
       download an apStar file
    INPUT:
       loc_id - location ID
       apogee_id - APOGEE ID of the star
       dr= return the path corresponding to this data release (general default)
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-01-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.apStarPath(loc_id,apogee_id,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                _dr_string(dr)),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr)
    return None

def modelSpec(lib='GK',teff=4500,logg=2.5,metals=0.,
              cfe=0.,nfe=0.,afe=0.,vmicro=2.,
              dr=None,rmHDU1=True,rmHDU2=True):
    """
    NAME:
       modelSpec
    PURPOSE:
       download a model spectrum file
    INPUT:
       lib= ('GK') spectral library
       teff= (4500) grid-point Teff
       logg= (2.5) grid-point logg
       metals= (0.) grid-point metallicity
       cfe= (0.) grid-point carbon-enhancement
       nfe= (0.) grid-point nitrogen-enhancement
       afe= (0.) grid-point alpha-enhancement
       vmicro= (2.) grid-point microturbulence
       dr= return the path corresponding to this data release
       rmHUD1= (True) if True, rm the first (v. large) HDU with the high-resolution model spectrum
       rmHDU2= (True) if True, rm the second (quite large) HDU with the model spectrum convolved with the LSF
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-01-20 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.modelSpecPath(lib=lib,teff=teff,logg=logg,metals=metals,
                                 cfe=cfe,nfe=nfe,afe=afe,vmicro=vmicro,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                _dr_string(dr)),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    # Post-processing of the file, removing the big first HDU or the first two for local storage
    if rmHDU1 or rmHDU2:
        # Open the file, need to use astropy's fits reader, bc the file has issues
        import astropy.io.fits as apyfits
        from astropy.utils.exceptions import AstropyUserWarning
        import warnings
        warnings.filterwarnings('ignore',category=AstropyUserWarning)
        hdulist= apyfits.open(filePath)
        if rmHDU1:
            hdu= apyfits.PrimaryHDU(numpy.zeros((2,2)))
        else:
            hdu= hdulist[0].copy()
        inp= [hdu]
        if rmHDU2:
            hdu2= apyfits.ImageHDU(numpy.zeros((2,2)))
        else:
            hdu2= hdulist[1].copy()
        inp.append(hdu2)
        inp.extend(hdulist[2:])
        newHdulist= apyfits.HDUList(inp)
        # Fix any issues in the headers
        for ii in range(5):
            if '[A/M]' in newHdulist[ii].header:
                newHdulist[ii].header['AM']= newHdulist[ii].header.pop('[A/M]',None)
            if '[C/M]' in newHdulist[ii].header:
                newHdulist[ii].header['CM']= newHdulist[ii].header.pop('[C/M]',None)
            if '[N/M]' in newHdulist[ii].header:
                newHdulist[ii].header['NM']= newHdulist[ii].header.pop('[N/M]',None)
        # Overwrite file
        newHdulist.writeto(filePath,clobber=True,output_verify='silentfix')
    return None

def ferreModelLibrary(lib='GK',pca=True,sixd=True,unf=False,dr=None,
                      convertToBin=True):
    """
    NAME:
       ferreModelLibrary
    PURPOSE:
       download a FERRE model library
    INPUT:
       lib= ('GK') spectral library
       dr= return the path corresponding to this data release
       pca= (True) if True, download the PCA compressed library
       sixd= (True) if True, download the 6D library (w/o vmicro)
       unf= (False) if True, download the binary library (otherwise ascii)
       convertToBin= (True) if True and not unf, convert the ascii file to binary using ferre's ascii2bin (which has to be on the path)
    OUTPUT:
       (none; just downloads; also downloads the corresponding .hdr)
    HISTORY:
       2015-01-21 - Written - Bovy (IAS)
    """
    if dr is None: dr= path._default_dr()
    # First make sure the file doesn't exist
    filePath= path.ferreModelLibraryPath(lib=lib,dr=dr,pca=pca,
                                         sixd=sixd,unf=unf)
    if not os.path.exists(filePath):
        # Create the file path
        downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                    _dr_string(dr)),
                                       _base_url(dr=dr))
        _download_file(downloadPath,filePath,dr,verbose=True)
        if convertToBin:
            sys.stdout.write('\r'+"Converting ascii model library to binary (can take a few minutes) ...\r")
            sys.stdout.flush()
            try:
                p= subprocess.Popen(['ascii2bin'],stdout=subprocess.PIPE,
                                    stdin=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=os.path.dirname(filePath))
                p.stdin.write(os.path.basename(filePath)+'\n')
                p.stdin.write('unf\n')
                stdout, stderr= p.communicate()
            except subprocess.CalledProcessError:
                print("Conversion of %s to binary failed ..." % (os.path.basename(filePath)))
            sys.stdout.write('\r'+_ERASESTR+'\r')
            sys.stdout.flush()
    # Also download the header
    if unf:
        headerFilePath= filePath.replace('.unf','.hdr')
    else:
        headerFilePath= filePath.replace('.dat','.hdr')
    if os.path.exists(headerFilePath): return None
    headerDownloadPath= headerFilePath.replace(os.path.join(path._APOGEE_DATA,
                                                            _dr_string(dr)),
                                               _base_url(dr=dr))
    _download_file(headerDownloadPath,headerFilePath,dr,verbose=True)
    return None

def modelAtmosphere(lib='kurucz_filled',teff=4500,logg=2.5,metals=0.,
                    cfe=0.,nfe=0.,afe=0.,vmicro=2.,dr=None):
    """
    NAME:
       modelAtmosphere
    PURPOSE:
       download a model atmosphere
    INPUT:
       lib= ('kurucz_filled') model atmosphere library
       teff= (4500) grid-point Teff
       logg= (2.5) grid-point logg
       metals= (0.) grid-point metallicity
       cfe= (0.) grid-point carbon-enhancement
       afe= (0.) grid-point alpha-enhancement
       vmicro= (2.) grid-point microturbulence
       dr= return the path corresponding to this data release
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-02-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= 'X'
    # First make sure the file doesn't exist
    filePath= path.modelAtmospherePath(lib=lib,teff=teff,logg=logg,
                                       metals=metals,cfe=cfe,afe=afe,
                                       vmicro=2.,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                _dr_string(dr)),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def linelist(linelist,dr=None):
    """
    NAME:
       linelist
    PURPOSE:
       download a linelist
    INPUT:
       linelist - name of the linelist
       dr= return the path corresponding to this data release
    OUTPUT:
       (none; just downloads)
    HISTORY:
       2015-02-13 - Written - Bovy (IAS)
    """
    if dr is None: dr= 'X'
    # First make sure the file doesn't exist
    filePath= path.linelistPath(linelist,dr=dr)
    if os.path.exists(filePath): return None
    # Create the file path
    downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
                                                _dr_string(dr)),
                                   _base_url(dr=dr))
    _download_file(downloadPath,filePath,dr,verbose=True)
    return None

def _download_file(downloadPath,filePath,dr,verbose=False):
    sys.stdout.write('\r'+"Downloading file %s ...\r" \
                         % (os.path.basename(filePath)))
    sys.stdout.flush()
    try:
        # make all intermediate directories
        os.makedirs(os.path.dirname(filePath))
    except OSError: pass
    # Safe way of downloading
    downloading= True
    interrupted= False
    file, tmp_savefilename= tempfile.mkstemp()
    os.close(file) #Easier this way
    ntries= 0
    while downloading:
        try:
            cmd= ['wget','%s' % downloadPath,
                  '-O','%s' % tmp_savefilename]
            if not verbose: cmd.append('-q')
            subprocess.check_call(cmd)
            shutil.move(tmp_savefilename,filePath)
            downloading= False
            if interrupted:
                raise KeyboardInterrupt
        except subprocess.CalledProcessError:
            if not downloading: #Assume KeyboardInterrupt
                raise
            elif ntries > _MAX_NTRIES:
                raise IOError('File %s does not appear to exist on the server ...' % (os.path.basename(filePath)))
            sys.stdout.write('\r'+"KeyboardInterrupt ignored while downloading ...\r")
            sys.stdout.flush()
            os.remove(tmp_savefilename)
            interrupted= True
            ntries+= 1
        finally:
            if os.path.exists(tmp_savefilename):
                os.remove(tmp_savefilename)
    sys.stdout.write('\r'+_ERASESTR+'\r')
    sys.stdout.flush()
    return None

def _base_url(dr,rc=False):
    if dr == '10': return _DR10_URL
    elif dr == '12': return _DR12_URL
    else: return _PROPRIETARY_URL

def _dr_string(dr):
    if dr == 'bosswork': return 'bosswork'
    else: return 'dr%s' % dr
<|endoftext|>


<commit_before>"""
sovrin-node package metadata
"""
__version_info__ = (0, 0, 3)
__version__ = '{}.{}.{}'.format(*__version_info__)
__author__ = "Sovrin Foundation."
__license__ = "Apache 2.0"

__all__ = ['__version_info__', '__version__', '__author__', '__license__']

__dependencies__ = {
    "sovrin_common": ">=0.0.3",
}
<commit_msg>advanced version number<commit_after>"""
sovrin-node package metadata
"""
__version_info__ = (0, 0, 4)
__version__ = '{}.{}.{}'.format(*__version_info__)
__author__ = "Sovrin Foundation."
__license__ = "Apache 2.0"

__all__ = ['__version_info__', '__version__', '__author__', '__license__']

__dependencies__ = {
    "sovrin_common": ">=0.0.4",
}
<|endoftext|>


<commit_before>import numpy as np
from autodiff import fmin_l_bfgs_b

def test_svm():

    rng = np.random.RandomState(1)

    # -- create some fake data
    x = rng.rand(10, 5)
    y = 2 * (rng.rand(10) > 0.5) - 1
    l2_regularization = 1e-4

    def loss_fn(weights, bias):
        margin = y * (np.dot(x, weights) + bias)
        loss = np.maximum(0, 1 - margin) ** 2
        l2_cost = 0.5 * l2_regularization * np.dot(weights, weights)
        loss = np.mean(loss) + l2_cost
        print 'ran loss_fn(), returning', loss
        return loss

    w, b = fmin_l_bfgs_b(loss_fn, [np.zeros(5), np.zeros(())])
    final_loss = loss_fn(w, b)
    assert np.allclose(final_loss, 0.7229)

<commit_msg>svm test case<commit_after>import numpy as np
from autodiff import fmin_l_bfgs_b

def test_svm():
    """
    This test case should match examples/linear_svm.py
    """

    rng = np.random.RandomState(1)

    # -- create some fake data
    x = rng.rand(10, 5)
    y = 2 * (rng.rand(10) > 0.5) - 1
    l2_regularization = 1e-4

    def loss_fn(weights, bias):
        margin = y * (np.dot(x, weights) + bias)
        loss = np.maximum(0, 1 - margin) ** 2
        l2_cost = 0.5 * l2_regularization * np.dot(weights, weights)
        loss = np.mean(loss) + l2_cost
        print 'ran loss_fn(), returning', loss
        return loss

    w, b = fmin_l_bfgs_b(loss_fn, [np.zeros(5), np.zeros(())])
    final_loss = loss_fn(w, b)
    assert np.allclose(final_loss, 0.7229)

<|endoftext|>


<commit_before># CTK: Cherokee Toolkit
#
# Authors:
#      Alvaro Lopez Ortega <alvaro@alobbs.com>
#
# Copyright (C) 2009 Alvaro Lopez Ortega
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of version 2 of the GNU General Public
# License as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
#

from RawHTML import RawHTML
from Widget import Widget
from Server import cfg

HEADER = [
    ''
]

class TextField (Widget):
    def __init__ (self, props=None):
        Widget.__init__ (self)
        self.type = "text"
        self.id   = 'widget%d'%(self.uniq_id)

        if props:
            self._props = props
        else:
            self._props = {}

        if not 'id' in self._props:
            self._props['id'] = self.id

    def __get_input_props (self):
        render = ''
        for prop in self._props:
            render += " %s" %(prop)
            value = self._props[prop]
            if value:
                render += '="%s"' %(str(value))
        return render

    def __get_error_div_props (self):
        render = ' id="error_%s"' % (self.id)
        if self._props.get('name'):
            render += ' key="%s"' %(self._props.get('name'))
        return render

    def Render (self):
        # Watermark
        js = ''
        if self._props.get('optional'):
            js += "$('#%s').DefaultValue('optional','%s');" %(self.id, _("optional"))

            if not self._props.get('class'):
                self._props['class'] = 'optional'
            else:
                self._props['class'] += ' optional'

        # Render the text field
        html = '<input type="%s"%s />' %(self.type, self.__get_input_props())

        # Render the error reporting field
        html += '

' %(self.__get_error_div_props())

        render = Widget.Render (self)
        render.html    += html
        render.js      += js
        render.headers += HEADER

        return render

    def JS_to_clean (self):
        return "$('#%s').attr('value', '');" %(self.id)
    def JS_to_focus (self):
        return "$('#%s').blur(); $('#%s').focus();" %(self.id, self.id)

class TextFieldPassword (TextField):
    def __init__ (self, *a, **kw):
        TextField.__init__ (self, *a, **kw)
        self.type = "password"

class TextCfg (TextField):
    def __init__ (self, key, optional=False, props={}):
        # Sanity checks
        assert type(key) == str
        assert type(optional) == bool
        assert type(props) in (type(None), dict)

        props = props.copy()

        # Read the key value
        val = cfg.get_val(key)
        if val:
            props['value'] = val

        if optional:
            props['optional'] = True

        # Other properties
        props['name'] = key

        # Init parent
        TextField.__init__ (self, props)

JS = """
$("#%(id)s")

.bind ("blur", function (event){
    var self  = $(this);
    var value = self.val();

    if (value != self.data('last_value')) {
       $("#activity").show();

       $.ajax ({type:     'POST',
                dataType: 'json',
                url:      '%(url)s',
                data:     '%(key)s='+value,
	 success: function (data) {
            self.data('last_value', value);

            var name = self.attr('name');
	    for (var key in data['updates']) {
               if (key == name) {
                  self.val (data['updates'][key]);
                  break;
               }
            }
         },
         error: function() {
            event.stopPropagation();
            self.val (self.data('last_value'));
         },
         complete: function (XMLHttpRequest, textStatus) {
	    $("#activity").fadeOut('fast');
         }
       });
    }
})

.data ('last_value', "%(value)s")

.bind ("keypress", function(event){
    if (event.keyCode == 13) {
        focus_next_input (this);
    }
});
"""

class TextCfgAuto (TextCfg):
    def __init__ (self, key, url, optional=False, props=None):
        self.key = key
        self.url = url
        TextCfg.__init__ (self, key, optional, props)

    def Render (self):
        value = cfg.get_val (self.key, '')

        render = TextCfg.Render(self)
        render.js += JS %({'id':    self.id,
                           'url':   self.url,
                           'key':   self.key,
                           'value': value})
        return render
<commit_msg>Adds a new parameter to CTK.TextField so a "optional" text can be specified from its constructor method.<commit_after># CTK: Cherokee Toolkit
#
# Authors:
#      Alvaro Lopez Ortega <alvaro@alobbs.com>
#
# Copyright (C) 2009 Alvaro Lopez Ortega
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of version 2 of the GNU General Public
# License as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
#

from RawHTML import RawHTML
from Widget import Widget
from Server import cfg

HEADER = [
    ''
]

class TextField (Widget):
    def __init__ (self, props=None):
        Widget.__init__ (self)
        self.type = "text"
        self.id   = 'widget%d'%(self.uniq_id)

        if props:
            self._props = props
        else:
            self._props = {}

        if not 'id' in self._props:
            self._props['id'] = self.id

    def __get_input_props (self):
        render = ''
        for prop in self._props:
            render += " %s" %(prop)
            value = self._props[prop]
            if value:
                render += '="%s"' %(str(value))
        return render

    def __get_error_div_props (self):
        render = ' id="error_%s"' % (self.id)
        if self._props.get('name'):
            render += ' key="%s"' %(self._props.get('name'))
        return render

    def Render (self):
        # Watermark
        js = ''

        if self._props.get('optional'):
            optional_string = self._props.get('optional_string', _("optional"))
            js += "$('#%s').DefaultValue('optional','%s');" %(self.id, optional_string)

            if not self._props.get('class'):
                self._props['class'] = 'optional'
            else:
                self._props['class'] += ' optional'

        # Render the text field
        html = '<input type="%s"%s />' %(self.type, self.__get_input_props())

        # Render the error reporting field
        html += '

' %(self.__get_error_div_props())

        render = Widget.Render (self)
        render.html    += html
        render.js      += js
        render.headers += HEADER

        return render

    def JS_to_clean (self):
        return "$('#%s').attr('value', '');" %(self.id)
    def JS_to_focus (self):
        return "$('#%s').blur(); $('#%s').focus();" %(self.id, self.id)

class TextFieldPassword (TextField):
    def __init__ (self, *a, **kw):
        TextField.__init__ (self, *a, **kw)
        self.type = "password"

class TextCfg (TextField):
    def __init__ (self, key, optional=False, props={}):
        # Sanity checks
        assert type(key) == str
        assert type(optional) == bool
        assert type(props) in (type(None), dict)

        props = props.copy()

        # Read the key value
        val = cfg.get_val(key)
        if val:
            props['value'] = val

        if optional:
            props['optional'] = True

        # Other properties
        props['name'] = key

        # Init parent
        TextField.__init__ (self, props)

JS = """
$("#%(id)s")

.bind ("blur", function (event){
    var self  = $(this);
    var value = self.val();

    if (value != self.data('last_value')) {
       $("#activity").show();

       $.ajax ({type:     'POST',
                dataType: 'json',
                url:      '%(url)s',
                data:     '%(key)s='+value,
	 success: function (data) {
            self.data('last_value', value);

            var name = self.attr('name');
	    for (var key in data['updates']) {
               if (key == name) {
                  self.val (data['updates'][key]);
                  break;
               }
            }
         },
         error: function() {
            event.stopPropagation();
            self.val (self.data('last_value'));
         },
         complete: function (XMLHttpRequest, textStatus) {
	    $("#activity").fadeOut('fast');
         }
       });
    }
})

.data ('last_value', "%(value)s")

.bind ("keypress", function(event){
    if (event.keyCode == 13) {
        focus_next_input (this);
    }
});
"""

class TextCfgAuto (TextCfg):
    def __init__ (self, key, url, optional=False, props=None):
        self.key = key
        self.url = url
        TextCfg.__init__ (self, key, optional, props)

    def Render (self):
        value = cfg.get_val (self.key, '')

        render = TextCfg.Render(self)
        render.js += JS %({'id':    self.id,
                           'url':   self.url,
                           'key':   self.key,
                           'value': value})
        return render
<|endoftext|>


<commit_before>from hashlib import md5, sha256

from speakerlib import niceify_number

import requests

from config import config

class EventRecorder(object):

    def __init__(self, db):

        self.db = db

    def post_to_slack(self, speech_text, **kwargs):
        if config.get("slack_url", None):
            requests.post(config["slack_url"], data=speech_text)

    def record_utterance(self, speech_text, record_utterance=False, event_result=None):
        if not record_utterance:
            return

        sha = sha256()
        sha.update(speech_text)
        sha_hash = sha.hexdigest()

        matched_snippet = self.db.execute("SELECT votes FROM snippets where sha256=?", [sha_hash]).fetchone()

        if matched_snippet:
            votes = matched_snippet["votes"] + 1
            self.db.execute("UPDATE snippets set votes=? where sha256=?", [votes, sha_hash])
        else:
            self.db.execute("INSERT into snippets (sha256, speech_text, votes) VALUES (?, ?, 0) ", [sha_hash, speech_text])

    def record_sound_event(self, sound_name, event_result=None):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [sound_name]).fetchone()

        if matched_sound:
            votes = matched_sound["votes"] + 1

            self.db.execute("UPDATE sounds set votes=? where name=?", [votes, sound_name])

    def queue_speech_for_tweet(self, speech_text, record_utterance=False, event_result=None):

        print record_utterance
        if not record_utterance:
            return

        if speech_text[0] == "!":
            return

        speech_text = speech_text[:139].replace("@","~")

        md5_hash = md5()
        md5_hash.update(speech_text)
        md5_hash = md5_hash.hexdigest()

        try:
            check_existence = self.db.execute("SELECT * FROM publish_queue where md5_hash=?", [md5_hash]).next()
        except StopIteration:
            check_existence = False

        if not check_existence:

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [speech_text, md5_hash])

    def queue_sound_for_tweet(self, name, event_result):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

        if matched_sound:
            text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
            md5_hash = md5()
            md5_hash.update(text)
            md5_hash = md5_hash.hexdigest()

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [text, md5_hash])

<commit_msg>Skip plugins for slack post<commit_after>from hashlib import md5, sha256

from speakerlib import niceify_number

import requests

from config import config

class EventRecorder(object):

    def __init__(self, db):

        self.db = db

    def post_to_slack(self, speech_text, **kwargs):

        #Skip plugins
        if speech_text[0] == "!":
            return

        if config.get("slack_url", None):
            requests.post(config["slack_url"], data=speech_text)

    def record_utterance(self, speech_text, record_utterance=False, event_result=None):
        if not record_utterance:
            return

        sha = sha256()
        sha.update(speech_text)
        sha_hash = sha.hexdigest()

        matched_snippet = self.db.execute("SELECT votes FROM snippets where sha256=?", [sha_hash]).fetchone()

        if matched_snippet:
            votes = matched_snippet["votes"] + 1
            self.db.execute("UPDATE snippets set votes=? where sha256=?", [votes, sha_hash])
        else:
            self.db.execute("INSERT into snippets (sha256, speech_text, votes) VALUES (?, ?, 0) ", [sha_hash, speech_text])

    def record_sound_event(self, sound_name, event_result=None):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [sound_name]).fetchone()

        if matched_sound:
            votes = matched_sound["votes"] + 1

            self.db.execute("UPDATE sounds set votes=? where name=?", [votes, sound_name])

    def queue_speech_for_tweet(self, speech_text, record_utterance=False, event_result=None):

        print record_utterance
        if not record_utterance:
            return

        if speech_text[0] == "!":
            return

        speech_text = speech_text[:139].replace("@","~")

        md5_hash = md5()
        md5_hash.update(speech_text)
        md5_hash = md5_hash.hexdigest()

        try:
            check_existence = self.db.execute("SELECT * FROM publish_queue where md5_hash=?", [md5_hash]).next()
        except StopIteration:
            check_existence = False

        if not check_existence:

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [speech_text, md5_hash])

    def queue_sound_for_tweet(self, name, event_result):

        matched_sound = self.db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

        if matched_sound:
            text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
            md5_hash = md5()
            md5_hash.update(text)
            md5_hash = md5_hash.hexdigest()

            self.db.execute("INSERT INTO publish_queue (tweet_text, md5_hash) VALUES (?, ?)", [text, md5_hash])

<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Classes for working with all automata, including Turing machines."""

import abc

import automata.base.exceptions as exceptions

class Automaton(metaclass=abc.ABCMeta):
    """An abstract base class for all automata, including Turing machines."""

    @abc.abstractmethod
    def __init__(self, obj=None, **kwargs):
        """Initialize a complete automaton."""
        raise NotImplementedError()

    @abc.abstractmethod
    def validate_self(self):
        """Return True if this automaton is internally consistent."""
        raise NotImplementedError()

    @abc.abstractmethod
    def _validate_input_yield(self, input_str):
        """Check if the given string is accepted by this automaton."""
        raise NotImplementedError()

    def _validate_input_return(self, input_str):
        """
        Check if the given string is accepted by this automaton.

        Return the automaton's final configuration if this string is valid.
        """
        validation_generator = self._validate_input_yield(input_str)
        for config in validation_generator:
            pass
        return config

    def validate_input(self, input_str, step=False):
        """
        Check if the given string is accepted by this automaton.

        If step is True, yield the configuration at each step. Otherwise,
        return the final configuration.
        """
        if step:
            return self._validate_input_yield(input_str)
        else:
            return self._validate_input_return(input_str)

    def _validate_initial_state(self):
        """Raise an error if the initial state is invalid."""
        if self.initial_state not in self.states:
            raise exceptions.InvalidStateError(
                '{} is not a valid initial state'.format(self.initial_state))

    def _validate_initial_state_transitions(self):
        """Raise an error if the initial state has no transitions defined."""
        if self.initial_state not in self.transitions:
            raise exceptions.MissingStateError(
                'initial state {} has no transitions defined'.format(
                    self.initial_state))

    def _validate_final_states(self):
        """Raise an error if any final states are invalid."""
        invalid_states = self.final_states - self.states
        if invalid_states:
            raise exceptions.InvalidStateError(
                'final states are not valid ({})'.format(
                    ', '.join(invalid_states)))

    def copy(self):
        """Create a deep copy of the automaton."""
        return self.__class__(**self.__dict__)

    def __eq__(self, other):
        """Check if two automata are equal."""
        return self.__dict__ == other.__dict__
<commit_msg>Clean up parameter list for Automata.__init__<commit_after>#!/usr/bin/env python3
"""Classes for working with all automata, including Turing machines."""

import abc

import automata.base.exceptions as exceptions

class Automaton(metaclass=abc.ABCMeta):
    """An abstract base class for all automata, including Turing machines."""

    @abc.abstractmethod
    def __init__(self):
        """Initialize a complete automaton."""
        raise NotImplementedError()

    @abc.abstractmethod
    def validate_self(self):
        """Return True if this automaton is internally consistent."""
        raise NotImplementedError()

    @abc.abstractmethod
    def _validate_input_yield(self, input_str):
        """Check if the given string is accepted by this automaton."""
        raise NotImplementedError()

    def _validate_input_return(self, input_str):
        """
        Check if the given string is accepted by this automaton.

        Return the automaton's final configuration if this string is valid.
        """
        validation_generator = self._validate_input_yield(input_str)
        for config in validation_generator:
            pass
        return config

    def validate_input(self, input_str, step=False):
        """
        Check if the given string is accepted by this automaton.

        If step is True, yield the configuration at each step. Otherwise,
        return the final configuration.
        """
        if step:
            return self._validate_input_yield(input_str)
        else:
            return self._validate_input_return(input_str)

    def _validate_initial_state(self):
        """Raise an error if the initial state is invalid."""
        if self.initial_state not in self.states:
            raise exceptions.InvalidStateError(
                '{} is not a valid initial state'.format(self.initial_state))

    def _validate_initial_state_transitions(self):
        """Raise an error if the initial state has no transitions defined."""
        if self.initial_state not in self.transitions:
            raise exceptions.MissingStateError(
                'initial state {} has no transitions defined'.format(
                    self.initial_state))

    def _validate_final_states(self):
        """Raise an error if any final states are invalid."""
        invalid_states = self.final_states - self.states
        if invalid_states:
            raise exceptions.InvalidStateError(
                'final states are not valid ({})'.format(
                    ', '.join(invalid_states)))

    def copy(self):
        """Create a deep copy of the automaton."""
        return self.__class__(**self.__dict__)

    def __eq__(self, other):
        """Check if two automata are equal."""
        return self.__dict__ == other.__dict__
<|endoftext|>