<commit_before>def send(data, pid, n=none):
	if (n is none):
		epy_i_send data to pid
	else:
		i=0
		while i<n:
			d=data[i]
			epy_i_send d to pid
			i=i+1

def recv(pid, n=none):
	if (n is none):
		retV=0
		epy_i_recv retV from pid
		return retV
	else:
		dim retV[n]
		i=0
		while i<n:
			v=0
			epy_i_recv v from pid
			retV[i]=v
			i=i+1
		return retV

def sendrecv(data, pid):
	retV=0
	epy_i_sendrecv data tofrom pid into retV
	return retV

def bcast(data, root):
	retV=0
	epy_i_bcast data from root into retV
	return retV

def reduce(data, operator):
	retV=0
	if operator=="sum":
		epy_i_reduce epy_i_sum data into retV
	elif operator=="min":
		epy_i_reduce epy_i_min data into retV
	elif operator=="max":
		epy_i_reduce epy_i_max data into retV
	elif operator=="prod":
		epy_i_reduce epy_i_prod data into retV
	else:
		print "Operator "+operator+" not found"
	return retV

def sync():
	epy_i_sync

def coreid():
	return epy_i_coreid

def numcores():
	return epy_i_numcores

def ishost():
	if epy_i_ishost:
		return true
	else:
		return false

def isdevice():
	if epy_i_isdevice:
		return true
	else:
		return false<commit_msg>Variable size comms in sendrecv call<commit_after>def send(data, pid, n=none):
	if (n is none):
		epy_i_send data to pid
	else:
		i=0
		while i<n:
			d=data[i]
			epy_i_send d to pid
			i=i+1

def recv(pid, n=none):
	if (n is none):
		retV=0
		epy_i_recv retV from pid
		return retV
	else:
		dim retV[n]
		i=0
		while i<n:
			v=0
			epy_i_recv v from pid
			retV[i]=v
			i=i+1
		return retV

def sendrecv(data, pid, n=none):
	if (n is none):
		retV=0
		epy_i_sendrecv data tofrom pid into retV
		return retV
	else:
		dim retV[n]
		i=0
		while i<n:
			v=0
			d=data[i]
			epy_i_sendrecv d tofrom pid into v
			retV[i]=v
			i=i+1
		return retV

def bcast(data, root):
	retV=0
	epy_i_bcast data from root into retV
	return retV

def reduce(data, operator):
	retV=0
	if operator=="sum":
		epy_i_reduce epy_i_sum data into retV
	elif operator=="min":
		epy_i_reduce epy_i_min data into retV
	elif operator=="max":
		epy_i_reduce epy_i_max data into retV
	elif operator=="prod":
		epy_i_reduce epy_i_prod data into retV
	else:
		print "Operator "+operator+" not found"
	return retV

def sync():
	epy_i_sync

def coreid():
	return epy_i_coreid

def numcores():
	return epy_i_numcores

def ishost():
	if epy_i_ishost:
		return true
	else:
		return false

def isdevice():
	if epy_i_isdevice:
		return true
	else:
		return false<|endoftext|>


<commit_before>""" Formula for building parallel """
from pakit import Archive, Recipe

class Parallel(Recipe):
    """
    GNU parallel executes shell jobs in parallel
    """
    def __init__(self):
        super(Parallel, self).__init__()
        self.homepage = 'http://www.gnu.org/software/parallel'
        self.repos = {
            'unstable': Archive('https://ftp.gnu.org/gnu/parallel/'
                                'parallel-20180322.tar.bz2',
                                hash='6389ad5318247ea28a8e9ddc9e69bc2713ae5c'
                                '19e3783edda81da54ff6356497')
        }
        self.repos['stable'] = self.repos['unstable']

    def build(self):
        self.cmd('./configure --prefix={prefix}')
        self.cmd('make install')

    def verify(self):
        lines = self.cmd('parallel --version').output()
        assert lines[0].find('GNU parallel') != -1
<commit_msg>Bump version.<commit_after>""" Formula for building parallel """
from pakit import Archive, Recipe

class Parallel(Recipe):
    """
    GNU parallel executes shell jobs in parallel
    """
    def __init__(self):
        super(Parallel, self).__init__()
        self.homepage = 'http://www.gnu.org/software/parallel'
        self.repos = {
            'unstable': Archive('https://ftp.gnu.org/gnu/parallel/'
                                'parallel-20181022.tar.bz2',
                                hash='2e84dee3556cbb8f6a3794f5b21549faffb132'
                                'db3fc68e2e95922963adcbdbec')

        }
        self.repos['stable'] = self.repos['unstable']

    def build(self):
        self.cmd('./configure --prefix={prefix}')
        self.cmd('make install')

    def verify(self):
        lines = self.cmd('parallel --version').output()
        assert lines[0].find('GNU parallel') != -1
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import argparse
import datetime
import os
import json
import logging
import sys

import boto3

from tabulate import tabulate

from s4 import sync
from s4 import utils
from s4.clients import local, s3

CONFIG_FOLDER_PATH = os.path.expanduser('~/.config/s4')
CONFIG_FILE_PATH = os.path.join(CONFIG_FOLDER_PATH, 'sync.conf')

def get_s3_client(target, aws_access_key_id, aws_secret_access_key, region_name):
    s3_uri = s3.parse_s3_uri(target)
    s3_client = boto3.client(
        's3',
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        region_name=region_name,
    )
    return s3.S3SyncClient(s3_client, s3_uri.bucket, s3_uri.key)

def get_local_client(target):
    return local.LocalSyncClient(target)

def main(arguments):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--log-level',
        default='INFO',
        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
    )

    subparsers = parser.add_subparsers(dest='command')
    sync_parser = subparsers.add_parser('sync')
    sync_parser.add_argument('targets', nargs='*')
    sync_parser.add_argument('--conflicts', default=None, choices=['1', '2', 'ignore'])

    subparsers.add_parser('targets')
    subparsers.add_parser('add')
    edit_parser = subparsers.add_parser('edit')
    edit_parser.add_argument('target')

    ls_parser = subparsers.add_parser('ls')
    ls_parser.add_argument('target')
    ls_parser.add_argument('--sort-by', '-s', choices=['key', 'local', 's3'], default='key')
    ls_parser.add_argument('--descending', '-d', action='store_true')
    ls_parser.add_argument(
        '--all', '-A',
        dest='show_all',
        action='store_true',
        help='show deleted files',
    )

    remove_parser = subparsers.add_parser('rm')
    remove_parser.add_argument('target')

    args = parser.parse_args(arguments)

    if args.log_level == 'DEBUG':
        log_format = '%(levelname)s:%(module)s:%(lineno)s %(message)s'
    else:
        log_format = '%(message)s'

    logging.basicConfig(format=log_format, level=args.log_level)

    # shut boto up
    logging.getLogger('boto3').setLevel(logging.CRITICAL)
    logging.getLogger('botocore').setLevel(logging.CRITICAL)
    logging.getLogger('nose').setLevel(logging.CRITICAL)
    logging.getLogger('s3transfer').setLevel(logging.CRITICAL)

    logger = logging.getLogger(__name__)
    logger.setLevel(args.log_level)

    config = get_config()

    try:
        if args.command == 'sync':
            sync_command(args, config, logger)
        elif args.command == 'targets':
            targets_command(args, config, logger)
        elif args.command == 'add':
            add_command(args, config, logger)
        elif args.command == 'edit':
            edit_command(args, config, logger)
        elif args.command == 'ls':
            ls_command(args, config, logger)
        elif args.command == 'rm':
            rm_command(args, config, logger)
        else:
            parser.print_help()
    except KeyboardInterrupt:
        pass

def get_config():
    if not os.path.exists(CONFIG_FILE_PATH):
        return {}

    with open(CONFIG_FILE_PATH, 'r') as fp:
        config = json.load(fp)
    return config

def set_config(config):
    if not os.path.exists(CONFIG_FOLDER_PATH):
        os.makedirs(CONFIG_FOLDER_PATH)

    with open(CONFIG_FILE_PATH, 'w') as fp:
        json.dump(config, fp, indent=4)

def get_clients(entry):
    target_1 = entry['local_folder']
    target_2 = entry['s3_uri']
    aws_access_key_id = entry['aws_access_key_id']
    aws_secret_access_key = entry['aws_secret_access_key']
    region_name = entry['region_name']

    # append trailing slashes to prevent incorrect prefix matching on s3
    if not target_1.endswith('/'):
        target_1 += '/'
    if not target_2.endswith('/'):
        target_2 += '/'

    client_1 = get_local_client(target_1)
    client_2 = get_s3_client(target_2, aws_access_key_id, aws_secret_access_key, region_name)
    return client_1, client_2

def sync_command(args, config, logger):
    all_targets = list(config['targets'].keys())
    if not args.targets:
        targets = all_targets
    else:
        targets = args.targets

    try:
        for name in sorted(targets):
            if name not in config['targets']:
                logger.info('"%s" is an unknown target. Choices are: %s', name, all_targets)
                continue

            entry = config['targets'][name]
            client_1, client_2 = get_clients(entry)

            worker = sync.SyncWorker(client_1, client_2)

            logger.info('Syncing %s [%s <=> %s]', name, client_1.get_uri(), client_2.get_uri())
            worker.sync(conflict_choice=args.conflicts)
    except KeyboardInterrupt:
        logger.warning('Quitting due to Keyboard Interrupt...')

def targets_command(args, config, logger):
    if 'targets' not in config:
        return

    for name in sorted(config['targets']):
        entry = config['targets'][name]
        logger.info('%s: [%s <=> %s]', name, entry['local_folder'], entry['s3_uri'])

def add_command(args, config, logger):
    entry = {}

    entry['local_folder'] = os.path.expanduser(utils.get_input('local folder: '))
    entry['s3_uri'] = utils.get_input('s3 uri: ')
    entry['aws_access_key_id'] = utils.get_input('AWS Access Key ID: ')
    entry['aws_secret_access_key'] = utils.get_input('AWS Secret Access Key: ', secret=True)
    entry['region_name'] = utils.get_input('region name: ')

    default_name = os.path.basename(entry['s3_uri'])
    name = utils.get_input('Provide a name for this entry [{}]: '.format(default_name))

    if not name:
        name = default_name

    if 'targets' not in config:
        config['targets'] = {}

    config['targets'][name] = entry

    set_config(config)

def edit_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        logger.info('Use the "add" command to do this')
        return

    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    entry = config['targets'][args.target]

    local_folder = entry.get('local_folder', '')
    s3_uri = entry.get('s3_uri', '')
    aws_access_key_id = entry.get('aws_access_key_id')
    aws_secret_access_key = entry.get('aws_secret_access_key')
    region_name = entry.get('region_name')

    new_local_folder = utils.get_input('local folder [{}]: '.format(local_folder))
    new_s3_uri = utils.get_input('s3 uri [{}]: '.format(s3_uri))
    new_aws_access_key_id = utils.get_input('AWS Access Key ID [{}]: '.format(aws_access_key_id))

    secret_key_prompt = 'AWS Secret Access Key [{}]: '.format(aws_secret_access_key)
    new_aws_secret_access_key = utils.get_input(secret_key_prompt, secret=True)

    new_region_name = utils.get_input('region name [{}]: '.format(region_name))

    if new_local_folder:
        entry['local_folder'] = new_local_folder
    if new_s3_uri:
        entry['s3_uri'] = new_s3_uri
    if new_aws_access_key_id:
        entry['aws_access_key_id'] = new_aws_access_key_id
    if new_aws_secret_access_key:
        entry['aws_secret_access_key'] = new_aws_secret_access_key
    if new_region_name:
        entry['region_name'] = new_region_name

    config['targets'][args.target] = entry

    set_config(config)

def ls_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        logger.info('Use the "add" command to do this')
        return
    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    target = config['targets'][args.target]
    client_1, client_2 = get_clients(target)

    sort_by = args.sort_by.lower()
    descending = args.descending

    keys = set(client_1.index) | set(client_2.index)

    data = []
    for key in sorted(keys):
        entry_1 = client_1.index.get(key, {})
        entry_2 = client_2.index.get(key, {})

        ts_1 = entry_1.get('local_timestamp')
        ts_2 = entry_2.get('local_timestamp')

        if args.show_all or ts_1 is not None:
            data.append((
                key,
                datetime.datetime.utcfromtimestamp(int(ts_1)) if ts_1 is not None else '<deleted>',
                datetime.datetime.utcfromtimestamp(int(ts_2)) if ts_2 is not None else None,
            ))

    headers = ['key', 'local', 's3']
    data = sorted(data, reverse=descending, key=lambda x: x[headers.index(sort_by)])

    logger.info(tabulate(data, headers=headers))

def rm_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        return
    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    del config['targets'][args.target]
    set_config(config)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Better program name for usage help<commit_after># -*- coding: utf-8 -*-

import argparse
import datetime
import os
import json
import logging
import sys

import boto3

from tabulate import tabulate

from s4 import sync
from s4 import utils
from s4.clients import local, s3

CONFIG_FOLDER_PATH = os.path.expanduser('~/.config/s4')
CONFIG_FILE_PATH = os.path.join(CONFIG_FOLDER_PATH, 'sync.conf')

def get_s3_client(target, aws_access_key_id, aws_secret_access_key, region_name):
    s3_uri = s3.parse_s3_uri(target)
    s3_client = boto3.client(
        's3',
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        region_name=region_name,
    )
    return s3.S3SyncClient(s3_client, s3_uri.bucket, s3_uri.key)

def get_local_client(target):
    return local.LocalSyncClient(target)

def main(arguments):
    parser = argparse.ArgumentParser(prog='s4')
    parser.add_argument(
        '--log-level',
        default='INFO',
        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
    )

    subparsers = parser.add_subparsers(dest='command')
    sync_parser = subparsers.add_parser('sync')
    sync_parser.add_argument('targets', nargs='*')
    sync_parser.add_argument('--conflicts', default=None, choices=['1', '2', 'ignore'])

    subparsers.add_parser('targets')
    subparsers.add_parser('add')
    edit_parser = subparsers.add_parser('edit')
    edit_parser.add_argument('target')

    ls_parser = subparsers.add_parser('ls')
    ls_parser.add_argument('target')
    ls_parser.add_argument('--sort-by', '-s', choices=['key', 'local', 's3'], default='key')
    ls_parser.add_argument('--descending', '-d', action='store_true')
    ls_parser.add_argument(
        '--all', '-A',
        dest='show_all',
        action='store_true',
        help='show deleted files',
    )

    remove_parser = subparsers.add_parser('rm')
    remove_parser.add_argument('target')

    args = parser.parse_args(arguments)

    if args.log_level == 'DEBUG':
        log_format = '%(levelname)s:%(module)s:%(lineno)s %(message)s'
    else:
        log_format = '%(message)s'

    logging.basicConfig(format=log_format, level=args.log_level)

    # shut boto up
    logging.getLogger('boto3').setLevel(logging.CRITICAL)
    logging.getLogger('botocore').setLevel(logging.CRITICAL)
    logging.getLogger('nose').setLevel(logging.CRITICAL)
    logging.getLogger('s3transfer').setLevel(logging.CRITICAL)

    logger = logging.getLogger(__name__)
    logger.setLevel(args.log_level)

    config = get_config()

    try:
        if args.command == 'sync':
            sync_command(args, config, logger)
        elif args.command == 'targets':
            targets_command(args, config, logger)
        elif args.command == 'add':
            add_command(args, config, logger)
        elif args.command == 'edit':
            edit_command(args, config, logger)
        elif args.command == 'ls':
            ls_command(args, config, logger)
        elif args.command == 'rm':
            rm_command(args, config, logger)
        else:
            parser.print_help()
    except KeyboardInterrupt:
        pass

def get_config():
    if not os.path.exists(CONFIG_FILE_PATH):
        return {}

    with open(CONFIG_FILE_PATH, 'r') as fp:
        config = json.load(fp)
    return config

def set_config(config):
    if not os.path.exists(CONFIG_FOLDER_PATH):
        os.makedirs(CONFIG_FOLDER_PATH)

    with open(CONFIG_FILE_PATH, 'w') as fp:
        json.dump(config, fp, indent=4)

def get_clients(entry):
    target_1 = entry['local_folder']
    target_2 = entry['s3_uri']
    aws_access_key_id = entry['aws_access_key_id']
    aws_secret_access_key = entry['aws_secret_access_key']
    region_name = entry['region_name']

    # append trailing slashes to prevent incorrect prefix matching on s3
    if not target_1.endswith('/'):
        target_1 += '/'
    if not target_2.endswith('/'):
        target_2 += '/'

    client_1 = get_local_client(target_1)
    client_2 = get_s3_client(target_2, aws_access_key_id, aws_secret_access_key, region_name)
    return client_1, client_2

def sync_command(args, config, logger):
    all_targets = list(config['targets'].keys())
    if not args.targets:
        targets = all_targets
    else:
        targets = args.targets

    try:
        for name in sorted(targets):
            if name not in config['targets']:
                logger.info('"%s" is an unknown target. Choices are: %s', name, all_targets)
                continue

            entry = config['targets'][name]
            client_1, client_2 = get_clients(entry)

            worker = sync.SyncWorker(client_1, client_2)

            logger.info('Syncing %s [%s <=> %s]', name, client_1.get_uri(), client_2.get_uri())
            worker.sync(conflict_choice=args.conflicts)
    except KeyboardInterrupt:
        logger.warning('Quitting due to Keyboard Interrupt...')

def targets_command(args, config, logger):
    if 'targets' not in config:
        return

    for name in sorted(config['targets']):
        entry = config['targets'][name]
        logger.info('%s: [%s <=> %s]', name, entry['local_folder'], entry['s3_uri'])

def add_command(args, config, logger):
    entry = {}

    entry['local_folder'] = os.path.expanduser(utils.get_input('local folder: '))
    entry['s3_uri'] = utils.get_input('s3 uri: ')
    entry['aws_access_key_id'] = utils.get_input('AWS Access Key ID: ')
    entry['aws_secret_access_key'] = utils.get_input('AWS Secret Access Key: ', secret=True)
    entry['region_name'] = utils.get_input('region name: ')

    default_name = os.path.basename(entry['s3_uri'])
    name = utils.get_input('Provide a name for this entry [{}]: '.format(default_name))

    if not name:
        name = default_name

    if 'targets' not in config:
        config['targets'] = {}

    config['targets'][name] = entry

    set_config(config)

def edit_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        logger.info('Use the "add" command to do this')
        return

    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    entry = config['targets'][args.target]

    local_folder = entry.get('local_folder', '')
    s3_uri = entry.get('s3_uri', '')
    aws_access_key_id = entry.get('aws_access_key_id')
    aws_secret_access_key = entry.get('aws_secret_access_key')
    region_name = entry.get('region_name')

    new_local_folder = utils.get_input('local folder [{}]: '.format(local_folder))
    new_s3_uri = utils.get_input('s3 uri [{}]: '.format(s3_uri))
    new_aws_access_key_id = utils.get_input('AWS Access Key ID [{}]: '.format(aws_access_key_id))

    secret_key_prompt = 'AWS Secret Access Key [{}]: '.format(aws_secret_access_key)
    new_aws_secret_access_key = utils.get_input(secret_key_prompt, secret=True)

    new_region_name = utils.get_input('region name [{}]: '.format(region_name))

    if new_local_folder:
        entry['local_folder'] = new_local_folder
    if new_s3_uri:
        entry['s3_uri'] = new_s3_uri
    if new_aws_access_key_id:
        entry['aws_access_key_id'] = new_aws_access_key_id
    if new_aws_secret_access_key:
        entry['aws_secret_access_key'] = new_aws_secret_access_key
    if new_region_name:
        entry['region_name'] = new_region_name

    config['targets'][args.target] = entry

    set_config(config)

def ls_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        logger.info('Use the "add" command to do this')
        return
    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    target = config['targets'][args.target]
    client_1, client_2 = get_clients(target)

    sort_by = args.sort_by.lower()
    descending = args.descending

    keys = set(client_1.index) | set(client_2.index)

    data = []
    for key in sorted(keys):
        entry_1 = client_1.index.get(key, {})
        entry_2 = client_2.index.get(key, {})

        ts_1 = entry_1.get('local_timestamp')
        ts_2 = entry_2.get('local_timestamp')

        if args.show_all or ts_1 is not None:
            data.append((
                key,
                datetime.datetime.utcfromtimestamp(int(ts_1)) if ts_1 is not None else '<deleted>',
                datetime.datetime.utcfromtimestamp(int(ts_2)) if ts_2 is not None else None,
            ))

    headers = ['key', 'local', 's3']
    data = sorted(data, reverse=descending, key=lambda x: x[headers.index(sort_by)])

    logger.info(tabulate(data, headers=headers))

def rm_command(args, config, logger):
    if 'targets' not in config:
        logger.info('You have not added any targets yet')
        return
    if args.target not in config['targets']:
        all_targets = sorted(list(config['targets'].keys()))
        logger.info('"%s" is an unknown target', args.target)
        logger.info('Choices are: %s', all_targets)
        return

    del config['targets'][args.target]
    set_config(config)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import requests
import dataset
import os
import sys
import re
import datetime as dt
import time
import traceback
from tqdm import tqdm
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
from dateutil.parser import parse as dtparse

# Number of seconds to sleep between crawls
CRAWL_TIMEOUT = 10

THIRTY_DAY_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews in the last 30 days')
ALL_TIME_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews for this game')
DETAILS_BOX_REGEX = re.compile(r'^Title: ([^\n]+)(?:\nGenre: ([^\n]+))?(?:\nDeveloper: ([^\n]+))?(?:\nPublisher: ([^\n]+))?')
NUM_ACHIEVEMENTS_REGEX = re.compile(r'Includes ([,[0-9]+) Steam Achievements')

def upsert_all_apps(db):
    '''
    Get the full list of steam apps and upsert them in our database
    on the basis of steam's app ID.
    '''
    json = requests.get('http://api.steampowered.com/ISteamApps/GetAppList/v0001/').json()

    apps = json['applist']['apps']['app']

    db.begin()

    for app in tqdm(apps):
        db['game'].upsert({
            'steam_app_id': app['appid'],
            'game_name': app['name'],
        }, keys=['app_id'])

    db.commit()

def scrape_store_page(driver, app_id):
    '''
    Extract all the information we can from the store page for a given app ID.

    Use the given driver so we don't have to worry about closing it when we exit.
    '''
    # TODO (maybe): add "ignore_reason" field to track why we skipped an app
    results = {'steam_app_id': app_id}
    store_base_url = "http://store.steampowered.com"
    app_url = "{}/app/{}".format(store_base_url, app_id)
    driver.get(app_url)

    try:
        # If this succeeds, we need to pass through the age gate.
        driver.find_element_by_id('agegate_box')

        select_element = driver.find_element_by_id('ageYear')
        # open year dialog
        select_element.click()
        # select correct year
        select_element.find_element_by_css_selector('option[value="1993"]').click()
        # close year dialog
        select_element.click()
        # submit the form
        driver.find_element_by_id('agecheck_form').submit()
    except NoSuchElementException:
        # No age gate; we're good to continue
        pass

    try:
        # If this succeeds, we need to click on the "continue" button to tell
        # steam we're okay with seeing NSFW content.
        driver.find_element_by_class_name('agegate_tags')

        # Click the "Continue" button
        (driver.find_element_by_css_selector(
                '.agegate_text_container.btns > a.btn_grey_white_innerfade:first-child'
            )
            .click())
    except NoSuchElementException:
        # No NSFW gate; we're good to continue
        pass

    if driver.current_url in (store_base_url, '{}/'.format(store_base_url)):
        # We were redirected; the app doesn't have a store page.
        return results
    elif 'store.steampowered.com/video' in driver.current_url:
        # This is a trailer for something else; we'll get the actual app later.
        return results

    try:
        # If this succeeds, the app has no store page; its store page
        # redirects to its community hub instead.  Skip it.
        driver.find_element_by_id('AppHubCards')
        return results
    except NoSuchElementException:
        pass

    try:
        # If this succeeds, we've got a steam store error
        error_element = driver.find_element_by_id('error_box')
        error_text = error_element.find_element_by_class_name('error')
        if error_text.text == 'This item is currently unavailable in your region':
            # We can't see this app; ignore it
            return results
    except NoSuchElementException:
        pass

    try:
        # If this succeeds, Chrome is showing us an error
        error_element = driver.find_element_by_class_name('error-code')
        if error_element.text == 'ERR_TOO_MANY_REDIRECTS':
            # Something wonky with the server response for this store page;
            # it's redirecting infinitely to itself.  Ignore it
            return results
    except NoSuchElementException:
        pass

    # Get the description first, since it tells us whether the app is streaming video
    # (which means we don't care about it)
    descriptions = driver.find_elements_by_class_name('game_area_description')
    for description in descriptions:
        if description.text.startswith('ABOUT THIS GAME'):
            results['is_dlc'] = False
            results['long_description'] = description.text
        elif description.text.startswith('ABOUT THIS CONTENT'):
            results['is_dlc'] = True
            results['long_description'] = description.text
        elif description.text.startswith('ABOUT THIS SERIES'):
            # This is streaming video; we don't care about it, so just return
            return results
        elif description.text.startswith('ABOUT THIS SOFTWARE'):
            # This is computer software; ignore it
            return results
    if 'long_description' not in results and len(descriptions) > 0:
        raise RuntimeError('Unable to parse description for app_id {}'.format(app_id))

    results['game_name'] = (driver
                            .find_element_by_class_name('apphub_AppName')
                            .text)

    try:
        results['short_description'] = (driver
                                        .find_element_by_class_name('game_description_snippet')
                                        .text)
    except NoSuchElementException:
        # DLC doesn't have a short description
        pass

    reviews_texts = [element.get_attribute('data-store-tooltip')
                    for element in (driver
                                    .find_elements_by_class_name('user_reviews_summary_row')
                                    )]

    for text in reviews_texts:
        thirty_day_match = THIRTY_DAY_REVIEW_REGEX.match(text)

        if thirty_day_match:
            results['pct_positive_reviews_last_30_days'] = int(thirty_day_match.group(1))
            results['reviews_last_30_days'] = int(thirty_day_match.group(2).replace(',', ''))
        else:
            all_time_match = ALL_TIME_REVIEW_REGEX.match(text)

            if all_time_match:
                results['pct_positive_reviews_all_time'] = int(all_time_match.group(1))
                results['reviews_all_time'] = int(all_time_match.group(2).replace(',', ''))

    try:
        results['release_date'] = dtparse(driver.find_element_by_css_selector('.release_date .date').text)
    except NoSuchElementException:
        # This app doesn't have a release date for some reason
        pass

    # There's additional detail about VR stuff here, but we're not worried about that for now
    details_text = driver.find_elements_by_css_selector('.details_block:not(.vrsupport)')[0].text
    details_match = DETAILS_BOX_REGEX.match(details_text)
    results['title'] = details_match.group(1)
    raw_genre = details_match.group(2)
    if raw_genre is not None:
        results['genres'] = raw_genre.split(', ')
    results['developer'] = details_match.group(3)
    results['publisher'] = details_match.group(4)

    block_titles_texts = [element.text
                        for element in (driver
                                        .find_elements_by_class_name('block_title')
                                        )]

    for text in block_titles_texts:
        num_achievements_match = NUM_ACHIEVEMENTS_REGEX.match(text)

        if num_achievements_match:
            results['num_achievements'] = int(num_achievements_match.group(1))

    try:
        raw_metacritic_score = driver.find_element_by_class_name('score').text
        if raw_metacritic_score != 'NA':
            results['metacritic_score'] = int(raw_metacritic_score)
    except NoSuchElementException:
        # Some games don't have metascores
        pass

    try:
        # Look for prices
        # NOTE: we'll take the first price available on the page (since
        # it's impossible to tell which one is for the actual game), so
        # if a game is only available in a package, we'll record its price
        # as the price of the package
        game_area = driver.find_element_by_class_name(
            'game_area_purchase_game'
        )
    except NoSuchElementException:
        # No price on the page
        game_area = None

    if game_area is not None:
        try:
            # Check within the "game_area" to avoid getting a DLC price
            raw_price = game_area.find_element_by_class_name('game_purchase_price').text
            if raw_price in ('Free to Play', 'Free'):
                price = 0
            elif raw_price == 'Third-party':
                # For all examples thus far, this has meant "free", but I don't think
                # we can assume that if the source is a 3rd party
                price = None
            else:
                price = float(raw_price.replace('$', ''))
            results['full_price'] = price
        except NoSuchElementException:
            # On sale
            raw_price = driver.find_element_by_class_name('discount_original_price').text
            results['full_price'] = float(raw_price.replace('$', ''))

    results['game_details'] = []
    game_details_elements = driver.find_elements_by_class_name('game_area_details_specs')
    for element in game_details_elements:
        results['game_details'].append(element.find_element_by_css_selector('a.name').text)

    results['tags'] = []
    try:
        # Try to get the big list of tags if it's there
        driver.find_element_by_css_selector('.app_tag.add_button').click()
        tag_elements = driver.find_elements_by_css_selector('#app_tagging_modal a.app_tag')
    except (NoSuchElementException, ElementNotVisibleException):
        # Settle for the short list if not
        tag_elements = driver.find_elements_by_css_selector('a.app_tag')
    for element in tag_elements:
        results['tags'].append(element.text)

    return results

def insert_with_mapping(*, db, descrs, entity_table, pk_name, join_table, mapping, app_id, crawl_time):
    '''
    Given a db connection, some list of description data, a table of entities, the name of the PK,
    a many-to-many join table for the entities to game crawls,
    a mapping from descrs to entity table IDs, and an app_id/crawl_time,
    update the entity table/mapping if necessary and insert
    the entity in the many-to-many join table.

    Mutates the mapping.
    '''
    for descr in descrs:
        try:
            entity_id = mapping[descr]
        except KeyError:
            entity_id = db[entity_table].insert({'descr': descr})
            mapping[descr] = entity_id

        db[join_table].insert({
            'steam_app_id': app_id,
            'crawl_time': crawl_time,
            pk_name: entity_id,
        })

def do_crawl(app_ids, db):
    '''
    Given a list of steam app IDs and a db connection, do a crawl for the app IDs
    and append the results to our list of crawls in the database.
    '''
    tag_mapping = {r['descr']: r['tag_id'] for r in db['steam_tag'].find()}
    detail_mapping = {r['descr']: r['detail_id'] for r in db['steam_game_detail'].find()}
    genre_mapping = {r['descr']: r['genre_id'] for r in db['steam_genre'].find()}

    # Add a handler here to allow us to gracefully save our work and quit
    # if the user halts a crawl early via Ctrl + C.
    # NOTE: since background jobs ignore SIGINT, this has no effect if
    # the script is launched in the background; see
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python#comment68802096_1112357
    should_quit = False

    # Set up a driver and re-use it so we don't have to worry about
    # closing it for each app
    driver = webdriver.Chrome()

    for app_id in tqdm(app_ids):
        try:
            db.begin()
            if should_quit:
                break

            time.sleep(CRAWL_TIMEOUT)
            results = scrape_store_page(driver, app_id)

            crawl_time = dt.datetime.now()
            results['crawl_time'] = crawl_time

            # Default to empty list if the results don't contain any of these
            tags, details, genres = [], [], []

            # Pull the lists off before we insert the main crawl record.
            if 'tags' in results:
                tags = results['tags']
                del results['tags']
            if 'game_details' in results:
                details = results['game_details']
                del results['game_details']
            if 'genres' in results:
                genres = results['genres']
                del results['genres']

            db['game_crawl'].insert(results)

            if len(tags) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=tags,
                    entity_table='steam_tag',
                    pk_name='tag_id',
                    join_table='game_crawl_tag',
                    mapping=tag_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(details) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=details,
                    entity_table='steam_game_detail',
                    pk_name='detail_id',
                    join_table='game_crawl_detail',
                    mapping=detail_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(genres) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=genres,
                    entity_table='steam_genre',
                    pk_name='genre_id',
                    join_table='game_crawl_genre',
                    mapping=genre_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )
            db.commit()
        except Exception as e:
            # Problem app; pass along our failure and continue to the next one
            print('Failed to load app ID {}; continuing'.format(app_id), file=sys.stderr)
            traceback.print_exc()

            # Ensure Postgres lets us continue by rolling back the current transaction
            db.rollback()
    driver.close()

def run():
    db = dataset.connect(os.environ['POSTGRES_URI'], ensure_schema=False)

    # Since we're not super worried about having an up-to-date list of apps,
    # run this only if the table is empty
    if db['game'].count() == 0:
        upsert_all_apps(db)

    # For now, just crawl the apps we don't already have
    missing_crawl_query = '''
    SELECT g.steam_app_id
    FROM game g
      LEFT JOIN game_crawl gc
        USING (steam_app_id)
    WHERE gc.steam_app_id IS NULL;
    '''

    missing_app_ids = [r['steam_app_id'] for r in db.query(missing_crawl_query)]

    do_crawl(missing_app_ids, db)

if __name__ == '__main__':
    run()
<commit_msg>Make "free-to-play" check case-insensitive<commit_after>import requests
import dataset
import os
import sys
import re
import datetime as dt
import time
import traceback
from tqdm import tqdm
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
from dateutil.parser import parse as dtparse

# Number of seconds to sleep between crawls
CRAWL_TIMEOUT = 10

THIRTY_DAY_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews in the last 30 days')
ALL_TIME_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews for this game')
DETAILS_BOX_REGEX = re.compile(r'^Title: ([^\n]+)(?:\nGenre: ([^\n]+))?(?:\nDeveloper: ([^\n]+))?(?:\nPublisher: ([^\n]+))?')
NUM_ACHIEVEMENTS_REGEX = re.compile(r'Includes ([,[0-9]+) Steam Achievements')

def upsert_all_apps(db):
    '''
    Get the full list of steam apps and upsert them in our database
    on the basis of steam's app ID.
    '''
    json = requests.get('http://api.steampowered.com/ISteamApps/GetAppList/v0001/').json()

    apps = json['applist']['apps']['app']

    db.begin()

    for app in tqdm(apps):
        db['game'].upsert({
            'steam_app_id': app['appid'],
            'game_name': app['name'],
        }, keys=['app_id'])

    db.commit()

def scrape_store_page(driver, app_id):
    '''
    Extract all the information we can from the store page for a given app ID.

    Use the given driver so we don't have to worry about closing it when we exit.
    '''
    # TODO (maybe): add "ignore_reason" field to track why we skipped an app
    results = {'steam_app_id': app_id}
    store_base_url = "http://store.steampowered.com"
    app_url = "{}/app/{}".format(store_base_url, app_id)
    driver.get(app_url)

    try:
        # If this succeeds, we need to pass through the age gate.
        driver.find_element_by_id('agegate_box')

        select_element = driver.find_element_by_id('ageYear')
        # open year dialog
        select_element.click()
        # select correct year
        select_element.find_element_by_css_selector('option[value="1993"]').click()
        # close year dialog
        select_element.click()
        # submit the form
        driver.find_element_by_id('agecheck_form').submit()
    except NoSuchElementException:
        # No age gate; we're good to continue
        pass

    try:
        # If this succeeds, we need to click on the "continue" button to tell
        # steam we're okay with seeing NSFW content.
        driver.find_element_by_class_name('agegate_tags')

        # Click the "Continue" button
        (driver.find_element_by_css_selector(
                '.agegate_text_container.btns > a.btn_grey_white_innerfade:first-child'
            )
            .click())
    except NoSuchElementException:
        # No NSFW gate; we're good to continue
        pass

    if driver.current_url in (store_base_url, '{}/'.format(store_base_url)):
        # We were redirected; the app doesn't have a store page.
        return results
    elif 'store.steampowered.com/video' in driver.current_url:
        # This is a trailer for something else; we'll get the actual app later.
        return results

    try:
        # If this succeeds, the app has no store page; its store page
        # redirects to its community hub instead.  Skip it.
        driver.find_element_by_id('AppHubCards')
        return results
    except NoSuchElementException:
        pass

    try:
        # If this succeeds, we've got a steam store error
        error_element = driver.find_element_by_id('error_box')
        error_text = error_element.find_element_by_class_name('error')
        if error_text.text == 'This item is currently unavailable in your region':
            # We can't see this app; ignore it
            return results
    except NoSuchElementException:
        pass

    try:
        # If this succeeds, Chrome is showing us an error
        error_element = driver.find_element_by_class_name('error-code')
        if error_element.text == 'ERR_TOO_MANY_REDIRECTS':
            # Something wonky with the server response for this store page;
            # it's redirecting infinitely to itself.  Ignore it
            return results
    except NoSuchElementException:
        pass

    # Get the description first, since it tells us whether the app is streaming video
    # (which means we don't care about it)
    descriptions = driver.find_elements_by_class_name('game_area_description')
    for description in descriptions:
        if description.text.startswith('ABOUT THIS GAME'):
            results['is_dlc'] = False
            results['long_description'] = description.text
        elif description.text.startswith('ABOUT THIS CONTENT'):
            results['is_dlc'] = True
            results['long_description'] = description.text
        elif description.text.startswith('ABOUT THIS SERIES'):
            # This is streaming video; we don't care about it, so just return
            return results
        elif description.text.startswith('ABOUT THIS SOFTWARE'):
            # This is computer software; ignore it
            return results
    if 'long_description' not in results and len(descriptions) > 0:
        raise RuntimeError('Unable to parse description for app_id {}'.format(app_id))

    results['game_name'] = (driver
                            .find_element_by_class_name('apphub_AppName')
                            .text)

    try:
        results['short_description'] = (driver
                                        .find_element_by_class_name('game_description_snippet')
                                        .text)
    except NoSuchElementException:
        # DLC doesn't have a short description
        pass

    reviews_texts = [element.get_attribute('data-store-tooltip')
                    for element in (driver
                                    .find_elements_by_class_name('user_reviews_summary_row')
                                    )]

    for text in reviews_texts:
        thirty_day_match = THIRTY_DAY_REVIEW_REGEX.match(text)

        if thirty_day_match:
            results['pct_positive_reviews_last_30_days'] = int(thirty_day_match.group(1))
            results['reviews_last_30_days'] = int(thirty_day_match.group(2).replace(',', ''))
        else:
            all_time_match = ALL_TIME_REVIEW_REGEX.match(text)

            if all_time_match:
                results['pct_positive_reviews_all_time'] = int(all_time_match.group(1))
                results['reviews_all_time'] = int(all_time_match.group(2).replace(',', ''))

    try:
        results['release_date'] = dtparse(driver.find_element_by_css_selector('.release_date .date').text)
    except NoSuchElementException:
        # This app doesn't have a release date for some reason
        pass

    # There's additional detail about VR stuff here, but we're not worried about that for now
    details_text = driver.find_elements_by_css_selector('.details_block:not(.vrsupport)')[0].text
    details_match = DETAILS_BOX_REGEX.match(details_text)
    results['title'] = details_match.group(1)
    raw_genre = details_match.group(2)
    if raw_genre is not None:
        results['genres'] = raw_genre.split(', ')
    results['developer'] = details_match.group(3)
    results['publisher'] = details_match.group(4)

    block_titles_texts = [element.text
                        for element in (driver
                                        .find_elements_by_class_name('block_title')
                                        )]

    for text in block_titles_texts:
        num_achievements_match = NUM_ACHIEVEMENTS_REGEX.match(text)

        if num_achievements_match:
            results['num_achievements'] = int(num_achievements_match.group(1))

    try:
        raw_metacritic_score = driver.find_element_by_class_name('score').text
        if raw_metacritic_score != 'NA':
            results['metacritic_score'] = int(raw_metacritic_score)
    except NoSuchElementException:
        # Some games don't have metascores
        pass

    try:
        # Look for prices
        # NOTE: we'll take the first price available on the page (since
        # it's impossible to tell which one is for the actual game), so
        # if a game is only available in a package, we'll record its price
        # as the price of the package
        game_area = driver.find_element_by_class_name(
            'game_area_purchase_game'
        )
    except NoSuchElementException:
        # No price on the page
        game_area = None

    if game_area is not None:
        try:
            # Check within the "game_area" to avoid getting a DLC price
            raw_price = game_area.find_element_by_class_name('game_purchase_price').text
            if raw_price.lower() in ('free to play', 'free'):
                price = 0
            elif raw_price == 'Third-party':
                # For all examples thus far, this has meant "free", but I don't think
                # we can assume that if the source is a 3rd party
                price = None
            else:
                price = float(raw_price.replace('$', ''))
            results['full_price'] = price
        except NoSuchElementException:
            # On sale
            raw_price = driver.find_element_by_class_name('discount_original_price').text
            results['full_price'] = float(raw_price.replace('$', ''))

    results['game_details'] = []
    game_details_elements = driver.find_elements_by_class_name('game_area_details_specs')
    for element in game_details_elements:
        results['game_details'].append(element.find_element_by_css_selector('a.name').text)

    results['tags'] = []
    try:
        # Try to get the big list of tags if it's there
        driver.find_element_by_css_selector('.app_tag.add_button').click()
        tag_elements = driver.find_elements_by_css_selector('#app_tagging_modal a.app_tag')
    except (NoSuchElementException, ElementNotVisibleException):
        # Settle for the short list if not
        tag_elements = driver.find_elements_by_css_selector('a.app_tag')
    for element in tag_elements:
        results['tags'].append(element.text)

    return results

def insert_with_mapping(*, db, descrs, entity_table, pk_name, join_table, mapping, app_id, crawl_time):
    '''
    Given a db connection, some list of description data, a table of entities, the name of the PK,
    a many-to-many join table for the entities to game crawls,
    a mapping from descrs to entity table IDs, and an app_id/crawl_time,
    update the entity table/mapping if necessary and insert
    the entity in the many-to-many join table.

    Mutates the mapping.
    '''
    for descr in descrs:
        try:
            entity_id = mapping[descr]
        except KeyError:
            entity_id = db[entity_table].insert({'descr': descr})
            mapping[descr] = entity_id

        db[join_table].insert({
            'steam_app_id': app_id,
            'crawl_time': crawl_time,
            pk_name: entity_id,
        })

def do_crawl(app_ids, db):
    '''
    Given a list of steam app IDs and a db connection, do a crawl for the app IDs
    and append the results to our list of crawls in the database.
    '''
    tag_mapping = {r['descr']: r['tag_id'] for r in db['steam_tag'].find()}
    detail_mapping = {r['descr']: r['detail_id'] for r in db['steam_game_detail'].find()}
    genre_mapping = {r['descr']: r['genre_id'] for r in db['steam_genre'].find()}

    # Add a handler here to allow us to gracefully save our work and quit
    # if the user halts a crawl early via Ctrl + C.
    # NOTE: since background jobs ignore SIGINT, this has no effect if
    # the script is launched in the background; see
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python#comment68802096_1112357
    should_quit = False

    # Set up a driver and re-use it so we don't have to worry about
    # closing it for each app
    driver = webdriver.Chrome()

    for app_id in tqdm(app_ids):
        try:
            db.begin()
            if should_quit:
                break

            time.sleep(CRAWL_TIMEOUT)
            results = scrape_store_page(driver, app_id)

            crawl_time = dt.datetime.now()
            results['crawl_time'] = crawl_time

            # Default to empty list if the results don't contain any of these
            tags, details, genres = [], [], []

            # Pull the lists off before we insert the main crawl record.
            if 'tags' in results:
                tags = results['tags']
                del results['tags']
            if 'game_details' in results:
                details = results['game_details']
                del results['game_details']
            if 'genres' in results:
                genres = results['genres']
                del results['genres']

            db['game_crawl'].insert(results)

            if len(tags) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=tags,
                    entity_table='steam_tag',
                    pk_name='tag_id',
                    join_table='game_crawl_tag',
                    mapping=tag_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(details) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=details,
                    entity_table='steam_game_detail',
                    pk_name='detail_id',
                    join_table='game_crawl_detail',
                    mapping=detail_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(genres) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=genres,
                    entity_table='steam_genre',
                    pk_name='genre_id',
                    join_table='game_crawl_genre',
                    mapping=genre_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )
            db.commit()
        except Exception as e:
            # Problem app; pass along our failure and continue to the next one
            print('Failed to load app ID {}; continuing'.format(app_id), file=sys.stderr)
            traceback.print_exc()

            # Ensure Postgres lets us continue by rolling back the current transaction
            db.rollback()
    driver.close()

def run():
    db = dataset.connect(os.environ['POSTGRES_URI'], ensure_schema=False)

    # Since we're not super worried about having an up-to-date list of apps,
    # run this only if the table is empty
    if db['game'].count() == 0:
        upsert_all_apps(db)

    # For now, just crawl the apps we don't already have
    missing_crawl_query = '''
    SELECT g.steam_app_id
    FROM game g
      LEFT JOIN game_crawl gc
        USING (steam_app_id)
    WHERE gc.steam_app_id IS NULL;
    '''

    missing_app_ids = [r['steam_app_id'] for r in db.query(missing_crawl_query)]

    do_crawl(missing_app_ids, db)

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before># Copyright (c) 2015, Anthony Schmieder
# Use of this source code is governed by the 2-clause BSD license that
# can be found in the LICENSE.txt file.

from __future__ import print_function

import argparse
import logging
import sys
import threading

from ui.checktracksapplicationwindow import CheckTracksApplicationWindow
from instrument import midi
from instrumentstate import InstrumentState
from sequencer.midiplayer import MidiPlayer
from playingsongstate import PlayingSongState

def parseArgs():
    parser = argparse.ArgumentParser(description='Keyzer track selection')
    parser.add_argument('--out-port', type=int, default=0,
                        help='Output MIDI port port number')
    parser.add_argument('--list-ports', action='store_true',
                        help='List available MIDI ports and terminate')
    parser.add_argument('--song-path', type=str,
                        help='Input MIDI file')
    parser.add_argument('--debug', action='store_true',
                        help='Print debug messages')
    args = parser.parse_args()

    if not args.list_ports and not args.song_path:
        parser.error("You must specify one of --song-path or --list-ports")

    return args

def probeMidiAndPrint():
    midiin = midi.InputConnection()
    inPorts = midiin.probeMidiPorts()
    midiout = midi.OutputConnection()
    outPorts = midiout.probeMidiPorts()
    printPort = lambda p: print("\t{}: {}".format(p.getNumber(), p.getName()))
    print("Input ports")
    map(printPort, inPorts)
    print("Output ports")
    map(printPort, outPorts)

def main(argv):
    args = parseArgs()

    level = logging.DEBUG if args.debug  else logging.WARNING
    logging.basicConfig(level=level)

    if args.list_ports:
        probeMidiAndPrint()
        sys.exit(0)

    midiout = midi.OutputConnection()
    outPorts = midiout.probeMidiPorts()
    midiout.openPort(outPorts[args.out_port])

    midiplayer = MidiPlayer(midiout, args.song_path)
    midiplayer.attach(PlayingSongState)

    noteSequence = midiplayer.getNoteSequence()
    app = CheckTracksApplicationWindow(noteSequence)

    playerThread = threading.Thread(target=midiplayer.play)
    playerThread.start()

    app.start()

    midiplayer.stop()
    playerThread.join()
    midiout.closePort()

if __name__ == "__main__":
    main(sys.argv)
    sys.exit(0)
<commit_msg>Remove input ports from checktracks --list-ports output<commit_after># Copyright (c) 2015, Anthony Schmieder
# Use of this source code is governed by the 2-clause BSD license that
# can be found in the LICENSE.txt file.

from __future__ import print_function

import argparse
import logging
import sys
import threading

from ui.checktracksapplicationwindow import CheckTracksApplicationWindow
from instrument import midi
from instrumentstate import InstrumentState
from sequencer.midiplayer import MidiPlayer
from playingsongstate import PlayingSongState

def parseArgs():
    parser = argparse.ArgumentParser(description='Keyzer track selection')
    parser.add_argument('--out-port', type=int, default=0,
                        help='Output MIDI port port number')
    parser.add_argument('--list-ports', action='store_true',
                        help='List available MIDI ports and terminate')
    parser.add_argument('--song-path', type=str,
                        help='Input MIDI file')
    parser.add_argument('--debug', action='store_true',
                        help='Print debug messages')
    args = parser.parse_args()

    if not args.list_ports and not args.song_path:
        parser.error("You must specify one of --song-path or --list-ports")

    return args

def probeMidiAndPrint():
    midiout = midi.OutputConnection()
    outPorts = midiout.probeMidiPorts()
    printPort = lambda p: print("\t{}: {}".format(p.getNumber(), p.getName()))
    print("Output ports")
    map(printPort, outPorts)

def main(argv):
    args = parseArgs()

    level = logging.DEBUG if args.debug  else logging.WARNING
    logging.basicConfig(level=level)

    if args.list_ports:
        probeMidiAndPrint()
        sys.exit(0)

    midiout = midi.OutputConnection()
    outPorts = midiout.probeMidiPorts()
    midiout.openPort(outPorts[args.out_port])

    midiplayer = MidiPlayer(midiout, args.song_path)
    midiplayer.attach(PlayingSongState)

    noteSequence = midiplayer.getNoteSequence()
    app = CheckTracksApplicationWindow(noteSequence)

    playerThread = threading.Thread(target=midiplayer.play)
    playerThread.start()

    app.start()

    midiplayer.stop()
    playerThread.join()
    midiout.closePort()

if __name__ == "__main__":
    main(sys.argv)
    sys.exit(0)
<|endoftext|>


<commit_before>from chef import Search
from chef.api import ChefAPI, autoconfigure
from chef.exceptions import ChefError

class Roledef(object):
    def __init__(self, name, api, hostname_attr, environment):
        self.name = name
        self.api = api
        self.hostname_attr = hostname_attr
        self.environment = environment

    def __call__(self):
        if map(int, self.api.version.split('.')) >= [0, 10]:
            query = 'roles:%s AND chef_environment:%s' % (self.name, self.environment)
        else:
            query = 'roles:%s' % self.name

        for row in Search('node', query, api=self.api):
            if row:
                yield row.object.attributes.get_dotted(self.hostname_attr)

def chef_roledefs(api=None, hostname_attr = 'fqdn', environment = '_default'):
    """Build a Fabric roledef dictionary from a Chef server.

    Example:

        from fabric.api import env, run, roles
        from chef.fabric import chef_roledefs

        env.roledefs = chef_roledefs()

        @roles('web_app')
        def mytask():
            run('uptime')

    hostname_attr is the attribute in the chef node that holds the real hostname.
    to refer to a nested attribute, separate the levels with '.'.
    for example 'ec2.public_hostname'
    environment is the chef environment whose nodes will be fetched. The default environment is "_default".
    """
    api = api or ChefAPI.get_global() or autoconfigure()
    if not api:
        raise ChefError('Unable to load Chef API configuration')
    roledefs = {}
    for row in Search('role', api=api):
        name = row['name']
        roledefs[name] =  Roledef(name, api, hostname_attr, environment)
    return roledefs
<commit_msg>Clean up and improve the degradation of the roledefs system for 0.9 compat.<commit_after>from chef.api import ChefAPI, autoconfigure
from chef.environment import Environment
from chef.exceptions import ChefError, ChefAPIVersionError
from chef.search import Search

class Roledef(object):
    def __init__(self, name, api, hostname_attr, environment=None):
        self.name = name
        self.api = api
        self.hostname_attr = hostname_attr
        self.environment = environment

    def __call__(self):
        query = 'roles:%s' % self.name
        if self.environment:
            query += ' AND chef_environment:%s' % self.environment

        for row in Search('node', query, api=self.api):
            if row:
                yield row.object.attributes.get_dotted(self.hostname_attr)

def chef_roledefs(api=None, hostname_attr = 'fqdn', environment = '_default'):
    """Build a Fabric roledef dictionary from a Chef server.

    Example:

        from fabric.api import env, run, roles
        from chef.fabric import chef_roledefs

        env.roledefs = chef_roledefs()

        @roles('web_app')
        def mytask():
            run('uptime')

    hostname_attr is the attribute in the chef node that holds the real hostname.
    to refer to a nested attribute, separate the levels with '.'.
    for example 'ec2.public_hostname'
    environment is the chef environment whose nodes will be fetched. The default environment is "_default".
    """
    api = api or ChefAPI.get_global() or autoconfigure()
    if not api:
        raise ChefError('Unable to load Chef API configuration')
    if api.version_parsed < Environment.api_version_parsed and environment is not None:
        raise ChefAPIVersionError('Environment support requires Chef API 0.10 or greater')
    roledefs = {}
    for row in Search('role', api=api):
        name = row['name']
        roledefs[name] =  Roledef(name, api, hostname_attr, environment)
    return roledefs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import chef  # pychef

class ChefWrapper(object):
    """ This class might be used to call Chef server API
        It can operate with different versions of Chef (11-12)
        and wraps calls of outdated 'pychef' library
    """

    def __init__(self, host, user, key=None, key_file=None, version=None, organization=None,
                 onlynodes=None, logger=None):
        self.host = host
        self.user = user
        self.key = key
        if not self.key:
            if not key_file:
                raise Exception("Either key or key_file should be passed!")
            with open(key_file, 'r') as kff:
                self.key = kff.read()
        self.version = str(version or "")
        self.organization = organization
        if self.version >= '12' and not organization:
            raise Exception("Parameter organization is required for Chef server version >= 12")
        self.logger = logger
        self.onlynodes = onlynodes or []

    def get_nodes(self):
        if self.version >= "12":
            chef_url = "https://%s/organizations/%s" % (self.host, self.organization)
        else:
            chef_url = "https://%s" % self.host
        # print(chef_url)
        all_nodes = []
        with chef.ChefAPI(chef_url, self.key, self.user):
            nodes = chef.Node.list()
        for nname, node in nodes.items():
            node_data = node.attributes.to_dict()
            if node_data:  # if not empty
                if onlynodes:
                    if not (node_data.get('hostname') in onlynodes or
                            node_data.get('fqdn') in onlynodes or
                            node_data.get('ipaddress') in onlynodes):
                        continue
                all_nodes.append(node_data)
        return all_nodes
<commit_msg>add pre-filter by nodes<commit_after># -*- coding: utf-8 -*-

import chef  # pychef

class ChefWrapper(object):
    """ This class might be used to call Chef server API
        It can operate with different versions of Chef (11-12)
        and wraps calls of outdated 'pychef' library
    """

    def __init__(self, host, user, key=None, key_file=None, version=None, organization=None,
                 onlynodes=None, logger=None):
        self.host = host
        self.user = user
        self.key = key
        if not self.key:
            if not key_file:
                raise Exception("Either key or key_file should be passed!")
            with open(key_file, 'r') as kff:
                self.key = kff.read()
        self.version = str(version or "")
        self.organization = organization
        if self.version >= '12' and not organization:
            raise Exception("Parameter organization is required for Chef server version >= 12")
        self.logger = logger
        self.onlynodes = onlynodes or []

    def get_nodes(self):
        if self.version >= "12":
            chef_url = "https://%s/organizations/%s" % (self.host, self.organization)
        else:
            chef_url = "https://%s" % self.host
        # print(chef_url)
        all_nodes = []
        with chef.ChefAPI(chef_url, self.key, self.user):
            nodes = chef.Node.list()
        for nname, node in nodes.items():
            node_data = node.attributes.to_dict()
            if node_data:  # if not empty
                if self.onlynodes:
                    if not (node_data.get('hostname') in self.onlynodes or
                            node_data.get('fqdn') in self.onlynodes or
                            node_data.get('ipaddress') in self.onlynodes):
                        continue
                all_nodes.append(node_data)
        return all_nodes
<|endoftext|>


<commit_before>from abc import abstractmethod, ABCMeta

from django.contrib.contenttypes.models import ContentType

from datetime import date

import people.tests.factories
from . import factories
from parties.tests.factories import PartyFactory

EXAMPLE_PARTIES = [
    {
        "ec_id": "PP53",
        "legacy_slug": "party:53",
        "name": "Labour Party",
        "attr": "labour_party",
        "register": "GB",
    },
    {
        "ec_id": "PP90",
        "legacy_slug": "party:90",
        "name": "Liberal Democrats",
        "attr": "ld_party",
        "register": "GB",
    },
    {
        "ec_id": "PP63",
        "legacy_slug": "party:63",
        "name": "Green Party",
        "attr": "green_party",
        "register": "GB",
    },
    {
        "ec_id": "PP52",
        "legacy_slug": "party:52",
        "name": "Conservative Party",
        "attr": "conservative_party",
        "register": "GB",
    },
    {
        "ec_id": "PP39",
        "legacy_slug": "party:39",
        "name": "Sinn Féin",
        "attr": "sinn_fein",
        "register": "NI",
    },
]

class UK2015ExamplesMixin(object, metaclass=ABCMeta):
    def setUp(self):
        ContentType.objects.clear_cache()

    @classmethod
    def setUpTestData(cls):
        super(UK2015ExamplesMixin, cls).setUpTestData()
        cls.gb_parties = factories.PartySetFactory.create(
            slug="gb", name="Great Britain"
        )
        cls.ni_parties = factories.PartySetFactory.create(
            slug="ni", name="Northern Ireland"
        )
        cls.commons = factories.ParliamentaryChamberFactory.create()
        # Create the 2010 and 2015 general elections:
        cls.election = factories.ElectionFactory.create(
            slug="parl.2015-05-07",
            name="2015 General Election",
            for_post_role="Member of Parliament",
            organization=cls.commons,
        )
        cls.earlier_election = factories.EarlierElectionFactory.create(
            slug="parl.2010-05-06",
            name="2010 General Election",
            for_post_role="Member of Parliament",
            current=False,
            organization=cls.commons,
        )
        # Create some example parties:
        PartyFactory.reset_sequence()

        for party in EXAMPLE_PARTIES:
            p = PartyFactory(
                ec_id=party["ec_id"],
                name=party["name"],
                legacy_slug=party["legacy_slug"],
                register=party["register"],
            )
            setattr(cls, party["attr"], p)

        # Create some example posts as well:
        EXAMPLE_CONSTITUENCIES = [
            {
                "id": "14419",
                "name": "Edinburgh East",
                "country": "Scotland",
                "attr": "edinburgh_east_post",
            },
            {
                "id": "14420",
                "name": "Edinburgh North and Leith",
                "country": "Scotland",
                "attr": "edinburgh_north_post",
            },
            {
                "id": "65808",
                "name": "Dulwich and West Norwood",
                "country": "England",
                "attr": "dulwich_post",
            },
            {
                "id": "65913",
                "name": "Camberwell and Peckham",
                "country": "England",
                "attr": "camberwell_post",
            },
        ]
        for cons in EXAMPLE_CONSTITUENCIES:
            label = "Member of Parliament for {}".format(cons["name"])
            post = factories.PostFactory.create(
                elections=(cls.election, cls.earlier_election),
                organization=cls.commons,
                slug=cons["id"],
                label=label,
                party_set=cls.gb_parties,
                group=cons["country"],
            )

            setattr(cls, cons["attr"], post)

            pee_attr_name = "{}_pee".format(cons["attr"])
            pee = post.postextraelection_set.get(election=cls.election)
            setattr(cls, pee_attr_name, pee)

            pee_attr_name = "{}_pee_earlier".format(cons["attr"])
            pee = post.postextraelection_set.get(election=cls.earlier_election)
            setattr(cls, pee_attr_name, pee)

        # Also create a local election and post:
        cls.local_council = factories.OrganizationFactory.create(
            name="Maidstone", slug="local-authority:maidstone"
        )
        cls.local_election = factories.ElectionFactory.create(
            slug="local.maidstone.2016-05-05",
            organization=cls.local_council,
            name="Maidstone local election",
            for_post_role="Local Councillor",
            election_date=date(2016, 5, 5),
        )
        cls.local_post = factories.PostFactory.create(
            elections=(cls.local_election,),
            slug="DIW:E05005004",
            label="Shepway South Ward",
            party_set=cls.gb_parties,
            organization=cls.local_council,
        )

    def create_lots_of_candidates(self, election, parties_and_counts):
        posts = [
            self.edinburgh_east_post,
            self.edinburgh_north_post,
            self.dulwich_post,
            self.camberwell_post,
        ]
        created = 0
        for party, candidates_to_create in parties_and_counts:
            for i in range(candidates_to_create):
                person_id = int("{}00{}".format(election.pk, created + 1))
                person = people.tests.factories.PersonFactory.create(
                    id=person_id, name="John Doe {}".format(person_id)
                )
                factories.MembershipFactory.create(
                    person=person,
                    post=posts[created % len(posts)],
                    party=party,
                    post_election=election.postextraelection_set.get(
                        post=posts[created % len(posts)]
                    ),
                )
                created += 1
<commit_msg>Set local_pee in test class<commit_after>from abc import abstractmethod, ABCMeta

from django.contrib.contenttypes.models import ContentType

from datetime import date

import people.tests.factories
from . import factories
from parties.tests.factories import PartyFactory

EXAMPLE_PARTIES = [
    {
        "ec_id": "PP53",
        "legacy_slug": "party:53",
        "name": "Labour Party",
        "attr": "labour_party",
        "register": "GB",
    },
    {
        "ec_id": "PP90",
        "legacy_slug": "party:90",
        "name": "Liberal Democrats",
        "attr": "ld_party",
        "register": "GB",
    },
    {
        "ec_id": "PP63",
        "legacy_slug": "party:63",
        "name": "Green Party",
        "attr": "green_party",
        "register": "GB",
    },
    {
        "ec_id": "PP52",
        "legacy_slug": "party:52",
        "name": "Conservative Party",
        "attr": "conservative_party",
        "register": "GB",
    },
    {
        "ec_id": "PP39",
        "legacy_slug": "party:39",
        "name": "Sinn Féin",
        "attr": "sinn_fein",
        "register": "NI",
    },
]

class UK2015ExamplesMixin(object, metaclass=ABCMeta):
    def setUp(self):
        ContentType.objects.clear_cache()

    @classmethod
    def setUpTestData(cls):
        super(UK2015ExamplesMixin, cls).setUpTestData()
        cls.gb_parties = factories.PartySetFactory.create(
            slug="gb", name="Great Britain"
        )
        cls.ni_parties = factories.PartySetFactory.create(
            slug="ni", name="Northern Ireland"
        )
        cls.commons = factories.ParliamentaryChamberFactory.create()
        # Create the 2010 and 2015 general elections:
        cls.election = factories.ElectionFactory.create(
            slug="parl.2015-05-07",
            name="2015 General Election",
            for_post_role="Member of Parliament",
            organization=cls.commons,
        )
        cls.earlier_election = factories.EarlierElectionFactory.create(
            slug="parl.2010-05-06",
            name="2010 General Election",
            for_post_role="Member of Parliament",
            current=False,
            organization=cls.commons,
        )
        # Create some example parties:
        PartyFactory.reset_sequence()

        for party in EXAMPLE_PARTIES:
            p = PartyFactory(
                ec_id=party["ec_id"],
                name=party["name"],
                legacy_slug=party["legacy_slug"],
                register=party["register"],
            )
            setattr(cls, party["attr"], p)

        # Create some example posts as well:
        EXAMPLE_CONSTITUENCIES = [
            {
                "id": "14419",
                "name": "Edinburgh East",
                "country": "Scotland",
                "attr": "edinburgh_east_post",
            },
            {
                "id": "14420",
                "name": "Edinburgh North and Leith",
                "country": "Scotland",
                "attr": "edinburgh_north_post",
            },
            {
                "id": "65808",
                "name": "Dulwich and West Norwood",
                "country": "England",
                "attr": "dulwich_post",
            },
            {
                "id": "65913",
                "name": "Camberwell and Peckham",
                "country": "England",
                "attr": "camberwell_post",
            },
        ]
        for cons in EXAMPLE_CONSTITUENCIES:
            label = "Member of Parliament for {}".format(cons["name"])
            post = factories.PostFactory.create(
                elections=(cls.election, cls.earlier_election),
                organization=cls.commons,
                slug=cons["id"],
                label=label,
                party_set=cls.gb_parties,
                group=cons["country"],
            )

            setattr(cls, cons["attr"], post)

            pee_attr_name = "{}_pee".format(cons["attr"])
            pee = post.postextraelection_set.get(election=cls.election)
            setattr(cls, pee_attr_name, pee)

            pee_attr_name = "{}_pee_earlier".format(cons["attr"])
            pee = post.postextraelection_set.get(election=cls.earlier_election)
            setattr(cls, pee_attr_name, pee)

        # Also create a local election and post:
        cls.local_council = factories.OrganizationFactory.create(
            name="Maidstone", slug="local-authority:maidstone"
        )
        cls.local_election = factories.ElectionFactory.create(
            slug="local.maidstone.2016-05-05",
            organization=cls.local_council,
            name="Maidstone local election",
            for_post_role="Local Councillor",
            election_date=date(2016, 5, 5),
        )
        cls.local_post = factories.PostFactory.create(
            elections=(cls.local_election,),
            slug="DIW:E05005004",
            label="Shepway South Ward",
            party_set=cls.gb_parties,
            organization=cls.local_council,
        )

        cls.local_pee = cls.local_post.postextraelection_set.get()

    def create_lots_of_candidates(self, election, parties_and_counts):
        posts = [
            self.edinburgh_east_post,
            self.edinburgh_north_post,
            self.dulwich_post,
            self.camberwell_post,
        ]
        created = 0
        for party, candidates_to_create in parties_and_counts:
            for i in range(candidates_to_create):
                person_id = int("{}00{}".format(election.pk, created + 1))
                person = people.tests.factories.PersonFactory.create(
                    id=person_id, name="John Doe {}".format(person_id)
                )
                factories.MembershipFactory.create(
                    person=person,
                    post=posts[created % len(posts)],
                    party=party,
                    post_election=election.postextraelection_set.get(
                        post=posts[created % len(posts)]
                    ),
                )
                created += 1
<|endoftext|>


<commit_before>from zou.app.models.notifications import Notification
from zou.app.utils import permissions

from .base import BaseModelResource, BaseModelsResource

class NotificationsResource(BaseModelsResource):

    def __init__(self):
        BaseModelsResource.__init__(self, Notification)

    def check_create_permissions(self, data):
        return permissions.check_admin_permissions()

class NotificationResource(BaseModelResource):

    def __init__(self):
        BaseModelResource.__init__(self, NotificationResource)

    def check_read_permissions(self, instance):
        return permissions.check_admin_permissions()

    def check_update_permissions(self, instance, data):
        return permissions.check_admin_permissions()

    def check_delete_permissions(self, instance):
        return permissions.check_admin_permissions()
<commit_msg>Fix crud controller for notifacation<commit_after>from zou.app.models.notifications import Notification
from zou.app.utils import permissions

from .base import BaseModelResource, BaseModelsResource

class NotificationsResource(BaseModelsResource):

    def __init__(self):
        BaseModelsResource.__init__(self, Notification)

    def check_create_permissions(self, data):
        return permissions.check_admin_permissions()

class NotificationResource(BaseModelResource):

    def __init__(self):
        BaseModelResource.__init__(self, Notification)

    def check_read_permissions(self, instance):
        return permissions.check_admin_permissions()

    def check_update_permissions(self, instance, data):
        return permissions.check_admin_permissions()

    def check_delete_permissions(self, instance):
        return permissions.check_admin_permissions()
<|endoftext|>


<commit_before>from __future__ import print_function
import os
import argparse
from git import Repo
import shutil
import json
import subprocess as sp
from collections import OrderedDict

#------------------------------------------------------------------------------------------------
# Argument Parser
#------------------------------------------------------------------------------------------------
parser = argparse.ArgumentParser(description="Get the species and associated genome builds available in ggd")
parser.add_argument("-rp", "--repo_path", help="The file path to the ggd-recipe repo")
parser.add_argument("-t", "--tmp_dir", required=True, help="The temporary directory to store the ggd metadata repo in")

args = parser.parse_args()

#------------------------------------------------------------------------------------------------
# File paths
#------------------------------------------------------------------------------------------------
GENOME_DIR = os.path.join(args.repo_path,"genomes")
RECIPE_DIR = os.path.join(args.repo_path,"recipes")
METADATA_GITHUB_URL = "git@github.com:gogetdata/ggd-metadata.git"

#------------------------------------------------------------------------------------------------
# Methods/Functions
#------------------------------------------------------------------------------------------------

def get_builds(species):
    """Get the genome_builds for a specific species

    get_builds
    ==========
    This method is used to get the genome builds associated with a specific species available in the ggd-recipes repo.

    Parameters:
    -----------
    1) species: The species in which to get the available genome builds for

    Returns:
    1) A list of available genome builds for the specified species
    """

    ## Get the species directory
    species_dir = os.path.join(GENOME_DIR,species)

    ## Make sure the directory exists
    assert os.path.isdir(species_dir)

    ## Get all available genome builds for that species
    return(os.listdir(species_dir))

#------------------------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------------------------

## Get unique species set
species_set = set(os.listdir(GENOME_DIR))

## Get genome builds for each species
species_dict = dict() ## Dictionary with key = species, value = list of genome builds
build_dict = dict() ## Dictionary with key = genome_build, value = species
for s in species_set:
    genome_builds = get_builds(s)
    species_dict[s] = genome_builds
    for b in genome_builds:
        build_dict[b] = s

## Get ggd channels
ggd_channels = {"channels":sorted(os.listdir(RECIPE_DIR))}

## Write json output
tmp_dir = os.path.join(os.getcwd(),"genome_metadata/")
print("\n-> Making a tmp directory to store genome metadata json files: %s" %tmp_dir)
if not os.path.exists(tmp_dir):
    os.makedirs(tmp_dir)

print("\n\t-> Writing species_to_build.json")
species_file_path = os.path.join(tmp_dir,"species_to_build.json")
with open(species_file_path, "w") as sf:
    json.dump(sorted(species_dict.items(), key=lambda t: t[0]),sf)

print("\n\t-> Writing build_to_species.json")
build_file_path = os.path.join(tmp_dir,"build_to_species.json")
with open(build_file_path, "w") as gf:
    json.dump(sorted(build_dict.items(), key=lambda t: t[0]),gf)

print("\n\t-> Writing ggd_channels.json")
ggd_channels_path = os.path.join(tmp_dir,"ggd_channels.json")
with open(ggd_channels_path, "w") as gc:
    json.dump(ggd_channels,gc)

## Get current working dir
cwd = os.getcwd()

## Make metadata_repo_dir
metadata_repo_dir = os.path.join(args.tmp_dir,"ggd-metadata")
if not os.path.isdir(metadata_repo_dir):
    os.makedirs(metadata_repo_dir)

Repo.clone_from(METADATA_GITHUB_URL,metadata_repo_dir)
os.chdir(metadata_repo_dir)
Repo(metadata_repo_dir).remotes.origin.pull()
repo = Repo(metadata_repo_dir)

## Check if species, builds, and channels are different
dest = os.path.join(metadata_repo_dir,"genome_metadata")

species_json_dict = {}
with open(os.path.join(dest,"species_to_build.json"), "r") as sj:
    species_json_dict = json.load(sj)

build_json_dict = {}
with open(os.path.join(dest,"build_to_species.json"), "r") as bj:
    build_json_dict = json.load(bj)

channels = {}
with open(os.path.join(dest,"ggd_channels.json"), "r") as gj:
    channels = json.load(gj)

### Add, commit, and push to ggd-metadata repo
sp.check_call(["git", "config", "user.email", "CIRCLECI@circleci.com"])
sp.check_call(["git", "config", "user.name", "CIRCLECI"])

commit = False
if sorted(species_json_dict.keys()) != sorted(species_dict.keys()):
    shutil.copy(species_file_path, dest)
    repo.git.add("genome_metadata/species_to_build.json")
    commit = True
    print("\n-> Add an updated species json file to ggd-metadata")

else:
    print("\n-> No change in species. The species json file will remain the same")

if sorted(build_json_dict.keys()) != sorted(build_dict.keys()):
    shutil.copy(build_file_path, dest)
    repo.git.add("genome_metadata/build_to_species.json")
    commit = True
    print("\n-> Add an updated geonme build json file to ggd-metadata")

else:
    print("\n-> No change in genome builds. The build json file will remain the same")

if sorted(channels["channels"] != sorted(ggd_channels["channels"]):
    shutil.copy(ggd_channels_path, dest)
    repo.git.add("genome_metadata/build_to_species.json")
    commit = True
    print("\n-> Add an updated ggd_channels json file to ggd-metadata")
else:
    print("\n-> No change in ggd channels. The ggd channels json file will remain the same")

if commit:
    from datetime import datetime
    date_time = datetime.today().isoformat()
    repo.git.commit("-m", "Update species_to_build.json and build_to_species.json files. ({date})".format(date=date_time))
    repo.git.push()
    print("\n-> Successfuly pushed new species and/or build json file(s) to ggd-metadata repo")

## Retrun to cwd
os.chdir(cwd)

<commit_msg>Update build_genomic_metadata_files.py<commit_after>from __future__ import print_function
import os
import argparse
from git import Repo
import shutil
import json
import subprocess as sp
from collections import OrderedDict

#------------------------------------------------------------------------------------------------
# Argument Parser
#------------------------------------------------------------------------------------------------
parser = argparse.ArgumentParser(description="Get the species and associated genome builds available in ggd")
parser.add_argument("-rp", "--repo_path", help="The file path to the ggd-recipe repo")
parser.add_argument("-t", "--tmp_dir", required=True, help="The temporary directory to store the ggd metadata repo in")

args = parser.parse_args()

#------------------------------------------------------------------------------------------------
# File paths
#------------------------------------------------------------------------------------------------
GENOME_DIR = os.path.join(args.repo_path,"genomes")
RECIPE_DIR = os.path.join(args.repo_path,"recipes")
METADATA_GITHUB_URL = "git@github.com:gogetdata/ggd-metadata.git"

#------------------------------------------------------------------------------------------------
# Methods/Functions
#------------------------------------------------------------------------------------------------

def get_builds(species):
    """Get the genome_builds for a specific species

    get_builds
    ==========
    This method is used to get the genome builds associated with a specific species available in the ggd-recipes repo.

    Parameters:
    -----------
    1) species: The species in which to get the available genome builds for

    Returns:
    1) A list of available genome builds for the specified species
    """

    ## Get the species directory
    species_dir = os.path.join(GENOME_DIR,species)

    ## Make sure the directory exists
    assert os.path.isdir(species_dir)

    ## Get all available genome builds for that species
    return(os.listdir(species_dir))

#------------------------------------------------------------------------------------------------
# Main
#------------------------------------------------------------------------------------------------

## Get unique species set
species_set = set(os.listdir(GENOME_DIR))

## Get genome builds for each species
species_dict = dict() ## Dictionary with key = species, value = list of genome builds
build_dict = dict() ## Dictionary with key = genome_build, value = species
for s in species_set:
    genome_builds = get_builds(s)
    species_dict[s] = genome_builds
    for b in genome_builds:
        build_dict[b] = s

## Get ggd channels
ggd_channels = {"channels":sorted(os.listdir(RECIPE_DIR))}

## Write json output
tmp_dir = os.path.join(os.getcwd(),"genome_metadata/")
print("\n-> Making a tmp directory to store genome metadata json files: %s" %tmp_dir)
if not os.path.exists(tmp_dir):
    os.makedirs(tmp_dir)

print("\n\t-> Writing species_to_build.json")
species_file_path = os.path.join(tmp_dir,"species_to_build.json")
with open(species_file_path, "w") as sf:
    json.dump(sorted(species_dict.items(), key=lambda t: t[0]),sf)

print("\n\t-> Writing build_to_species.json")
build_file_path = os.path.join(tmp_dir,"build_to_species.json")
with open(build_file_path, "w") as gf:
    json.dump(sorted(build_dict.items(), key=lambda t: t[0]),gf)

print("\n\t-> Writing ggd_channels.json")
ggd_channels_path = os.path.join(tmp_dir,"ggd_channels.json")
with open(ggd_channels_path, "w") as gc:
    json.dump(ggd_channels,gc)

## Get current working dir
cwd = os.getcwd()

## Make metadata_repo_dir
metadata_repo_dir = os.path.join(args.tmp_dir,"ggd-metadata")
if not os.path.isdir(metadata_repo_dir):
    os.makedirs(metadata_repo_dir)

Repo.clone_from(METADATA_GITHUB_URL,metadata_repo_dir)
os.chdir(metadata_repo_dir)
Repo(metadata_repo_dir).remotes.origin.pull()
repo = Repo(metadata_repo_dir)

## Check if species, builds, and channels are different
dest = os.path.join(metadata_repo_dir,"genome_metadata")

species_json_dict = {}
with open(os.path.join(dest,"species_to_build.json"), "r") as sj:
    species_json_dict = json.load(sj)

build_json_dict = {}
with open(os.path.join(dest,"build_to_species.json"), "r") as bj:
    build_json_dict = json.load(bj)

channels = {}
with open(os.path.join(dest,"ggd_channels.json"), "r") as gj:
    channels = json.load(gj)

### Add, commit, and push to ggd-metadata repo
sp.check_call(["git", "config", "user.email", "CIRCLECI@circleci.com"])
sp.check_call(["git", "config", "user.name", "CIRCLECI"])

commit = False
if sorted(species_json_dict.keys()) != sorted(species_dict.keys()):
    shutil.copy(species_file_path, dest)
    repo.git.add("genome_metadata/species_to_build.json")
    commit = True
    print("\n-> Add an updated species json file to ggd-metadata")

else:
    print("\n-> No change in species. The species json file will remain the same")

if sorted(build_json_dict.keys()) != sorted(build_dict.keys()):
    shutil.copy(build_file_path, dest)
    repo.git.add("genome_metadata/build_to_species.json")
    commit = True
    print("\n-> Add an updated geonme build json file to ggd-metadata")

else:
    print("\n-> No change in genome builds. The build json file will remain the same")

if sorted(channels["channels"]) != sorted(ggd_channels["channels"]):
    shutil.copy(ggd_channels_path, dest)
    repo.git.add("genome_metadata/build_to_species.json")
    commit = True
    print("\n-> Add an updated ggd_channels json file to ggd-metadata")
else:
    print("\n-> No change in ggd channels. The ggd channels json file will remain the same")

if commit:
    from datetime import datetime
    date_time = datetime.today().isoformat()
    repo.git.commit("-m", "Update species_to_build.json and build_to_species.json files. ({date})".format(date=date_time))
    repo.git.push()
    print("\n-> Successfuly pushed new species and/or build json file(s) to ggd-metadata repo")

## Retrun to cwd
os.chdir(cwd)

<|endoftext|>