<commit_before>import datetime
import logging

import geoip2.database
from django.conf import settings
from django.core.management import call_command
from django.db.models import Sum
from django.utils.timezone import now
from django_rq import job
from federation.hostmeta import fetchers
from federation.utils.network import fetch_host_ip

from thefederation.enums import Relay
from thefederation.models import Node, Platform, Protocol, Service, Stat

logger = logging.getLogger(__name__)

METHODS = ['mastodon', 'nodeinfo2', 'nodeinfo', 'matrix', 'misskey', 'statisticsjson']

def aggregate_daily_stats(date=None):
    if not date:
        date = now().date()
    # Do all platforms, protocols and then global
    totals = {
        'users_total': 0,
        'users_half_year': 0,
        'users_monthly': 0,
        'users_weekly': 0,
        'local_posts': 0,
        'local_comments': 0,
    }
    for platform in Platform.objects.all():
        stats = Stat.objects.exclude(
            node__last_success__lt=now() - datetime.timedelta(days=30)
        ).filter(
            node__platform=platform,
            date=date,
        ).aggregate(
            users_total=Sum('users_total'),
            users_half_year=Sum('users_half_year'),
            users_monthly=Sum('users_monthly'),
            users_weekly=Sum('users_weekly'),
            local_posts=Sum('local_posts'),
            local_comments=Sum('local_comments'),
        )
        Stat.objects.update_or_create(
            date=date, protocol=None, platform=platform, node=None, defaults=stats,
        )
        # Increment globals
        for key in totals:
            totals[key] += stats[key] if stats[key] else 0
    for protocol in Protocol.objects.all():
        stats = Stat.objects.exclude(
            node__last_success__lt=now() - datetime.timedelta(days=30)
        ).filter(
            node__protocols=protocol,
            date=date,
        ).aggregate(
            users_total=Sum('users_total'),
            users_half_year=Sum('users_half_year'),
            users_monthly=Sum('users_monthly'),
            users_weekly=Sum('users_weekly'),
            local_posts=Sum('local_posts'),
            local_comments=Sum('local_comments'),
        )
        Stat.objects.update_or_create(
            date=date, protocol=protocol, platform=None, node=None, defaults=stats,
        )
    # Add global stat
    Stat.objects.update_or_create(date=date, protocol=None, platform=None, node=None, defaults=totals)

def clean_duplicate_nodes():
    """
    Call the clean dupe nodes command.
    """
    call_command('clean_dupe_nodes')
    # Also re-aggregate stats from a few days
    for single_date in (now().date() - datetime.timedelta(n) for n in range(2)):
        aggregate_daily_stats(single_date)

def fetch_using_method(host, method):
    if method is None:
        return
    logger.debug(f'Fetching {host} using method {method}')
    func = getattr(fetchers, f"fetch_{method}_document")
    return func(host)

def fetch_node(host):
    """
    Fetch different documents in order

    If host exists, use preferred document, falling back to all.

    :param host: str
    :return: dict
    """
    # Use preferred method if known
    try:
        node = Node.objects.only('platform', 'version').get(host=host)
    except Node.DoesNotExist:
        methods = METHODS[:]
    else:
        result = fetch_using_method(host, node.preferred_method)
        if result:
            return result
        methods = METHODS[:]
        if node.preferred_method:
            methods.remove(node.preferred_method)

    # Use remaining methods
    for method in methods:
        result = fetch_using_method(host, method)
        if result:
            return result

def fill_country_information():
    logger.info('Updating country and IP information for all nodes.')
    ipdb = geoip2.database.Reader(settings.MAXMIND_DB_PATH)
    updates = 0
    for node in Node.objects.only('host', 'ip', 'country').active():
        try:
            save = False
            ip = fetch_host_ip(node.host)
            if node.ip != ip:
                node.ip = ip
                save = True
            if ip:
                response = ipdb.country(ip)
                if response.country and (not node.country or node.country.code != response.country.iso_code):
                    node.country = response.country.iso_code or ''
                    save = True
            if save:
                node.save()
                updates += 1
        except Exception as ex:
            logger.warning(f"Error trying to fill country info: {ex}")
    logger.info(f'Update of country and IP information done, updated {updates} nodes.')

@job('medium')
def poll_node(host):
    result = fetch_node(host)
    if not result:
        logger.info(f'No result for {host}.')
        return False

    assert host == result.get('host')
    platform, _created = Platform.objects.get_or_create(name=result['platform'])
    node, _created = Node.objects.update_or_create(
        host=host,
        defaults={
            'features': result.get('features', {}),
            'last_success': now(),
            'name': result.get('name') or host,
            'open_signups': result.get('open_signups', False),
            'organization_account': result.get('organization', {}).get('account', ''),
            'organization_contact': result.get('organization', {}).get('contact', ''),
            'organization_name': result.get('organization', {}).get('name', ''),
            'relay': result.get('relay') or Relay.NONE,
            'server_meta': result.get('server_meta', {}),
            'version': result.get('version', ''),
            'platform': platform,
        }
    )

    protocols = set()
    for protocol in result.get('protocols', []):
        assert protocol != ""
        if protocol == 'friendica':
            protocol = 'dfrn'
        elif protocol == 'gnusocial':
            protocol = 'ostatus'
        proto, _created = Protocol.objects.get_or_create(name=protocol)
        protocols.add(proto)
    node.protocols.set(protocols)
    services = set()
    for service in result.get('services', []):
        assert service != ""
        serv, _created = Service.objects.get_or_create(name=service)
        services.add(serv)
    node.services.set(services)

    activity = result.get('activity', {})
    users = activity.get('users', {})
    Stat.objects.update_or_create(
        node=node,
        date=now().date(),
        defaults={
            'users_total': users.get('total'),
            'users_half_year': users.get('half_year'),
            'users_monthly': users.get('monthly'),
            'users_weekly': users.get('weekly'),
            'local_posts': activity.get('local_posts'),
            'local_comments': activity.get('local_comments'),
        },
    )

    logger.info(f'Updated {host} successfully.')
    return True

def poll_nodes():
    logger.info(f'Queueing polling all nodes.')
    nodes_qs = Node.objects.only('host').active()
    for node in nodes_qs:
        poll_node.delay(node.host)
<commit_msg>Change the order of methods being run on new nodes<commit_after>import datetime
import logging

import geoip2.database
from django.conf import settings
from django.core.management import call_command
from django.db.models import Sum
from django.utils.timezone import now
from django_rq import job
from federation.hostmeta import fetchers
from federation.utils.network import fetch_host_ip

from thefederation.enums import Relay
from thefederation.models import Node, Platform, Protocol, Service, Stat

logger = logging.getLogger(__name__)

METHODS = ['nodeinfo2', 'nodeinfo', 'mastodon', 'matrix', 'misskey', 'statisticsjson']

def aggregate_daily_stats(date=None):
    if not date:
        date = now().date()
    # Do all platforms, protocols and then global
    totals = {
        'users_total': 0,
        'users_half_year': 0,
        'users_monthly': 0,
        'users_weekly': 0,
        'local_posts': 0,
        'local_comments': 0,
    }
    for platform in Platform.objects.all():
        stats = Stat.objects.exclude(
            node__last_success__lt=now() - datetime.timedelta(days=30)
        ).filter(
            node__platform=platform,
            date=date,
        ).aggregate(
            users_total=Sum('users_total'),
            users_half_year=Sum('users_half_year'),
            users_monthly=Sum('users_monthly'),
            users_weekly=Sum('users_weekly'),
            local_posts=Sum('local_posts'),
            local_comments=Sum('local_comments'),
        )
        Stat.objects.update_or_create(
            date=date, protocol=None, platform=platform, node=None, defaults=stats,
        )
        # Increment globals
        for key in totals:
            totals[key] += stats[key] if stats[key] else 0
    for protocol in Protocol.objects.all():
        stats = Stat.objects.exclude(
            node__last_success__lt=now() - datetime.timedelta(days=30)
        ).filter(
            node__protocols=protocol,
            date=date,
        ).aggregate(
            users_total=Sum('users_total'),
            users_half_year=Sum('users_half_year'),
            users_monthly=Sum('users_monthly'),
            users_weekly=Sum('users_weekly'),
            local_posts=Sum('local_posts'),
            local_comments=Sum('local_comments'),
        )
        Stat.objects.update_or_create(
            date=date, protocol=protocol, platform=None, node=None, defaults=stats,
        )
    # Add global stat
    Stat.objects.update_or_create(date=date, protocol=None, platform=None, node=None, defaults=totals)

def clean_duplicate_nodes():
    """
    Call the clean dupe nodes command.
    """
    call_command('clean_dupe_nodes')
    # Also re-aggregate stats from a few days
    for single_date in (now().date() - datetime.timedelta(n) for n in range(2)):
        aggregate_daily_stats(single_date)

def fetch_using_method(host, method):
    if method is None:
        return
    logger.debug(f'Fetching {host} using method {method}')
    func = getattr(fetchers, f"fetch_{method}_document")
    return func(host)

def fetch_node(host):
    """
    Fetch different documents in order

    If host exists, use preferred document, falling back to all.

    :param host: str
    :return: dict
    """
    # Use preferred method if known
    try:
        node = Node.objects.only('platform', 'version').get(host=host)
    except Node.DoesNotExist:
        methods = METHODS[:]
    else:
        result = fetch_using_method(host, node.preferred_method)
        if result:
            return result
        methods = METHODS[:]
        if node.preferred_method:
            methods.remove(node.preferred_method)

    # Use remaining methods
    for method in methods:
        result = fetch_using_method(host, method)
        if result:
            return result

def fill_country_information():
    logger.info('Updating country and IP information for all nodes.')
    ipdb = geoip2.database.Reader(settings.MAXMIND_DB_PATH)
    updates = 0
    for node in Node.objects.only('host', 'ip', 'country').active():
        try:
            save = False
            ip = fetch_host_ip(node.host)
            if node.ip != ip:
                node.ip = ip
                save = True
            if ip:
                response = ipdb.country(ip)
                if response.country and (not node.country or node.country.code != response.country.iso_code):
                    node.country = response.country.iso_code or ''
                    save = True
            if save:
                node.save()
                updates += 1
        except Exception as ex:
            logger.warning(f"Error trying to fill country info: {ex}")
    logger.info(f'Update of country and IP information done, updated {updates} nodes.')

@job('medium')
def poll_node(host):
    result = fetch_node(host)
    if not result:
        logger.info(f'No result for {host}.')
        return False

    assert host == result.get('host')
    platform, _created = Platform.objects.get_or_create(name=result['platform'])
    node, _created = Node.objects.update_or_create(
        host=host,
        defaults={
            'features': result.get('features', {}),
            'last_success': now(),
            'name': result.get('name') or host,
            'open_signups': result.get('open_signups', False),
            'organization_account': result.get('organization', {}).get('account', ''),
            'organization_contact': result.get('organization', {}).get('contact', ''),
            'organization_name': result.get('organization', {}).get('name', ''),
            'relay': result.get('relay') or Relay.NONE,
            'server_meta': result.get('server_meta', {}),
            'version': result.get('version', ''),
            'platform': platform,
        }
    )

    protocols = set()
    for protocol in result.get('protocols', []):
        assert protocol != ""
        if protocol == 'friendica':
            protocol = 'dfrn'
        elif protocol == 'gnusocial':
            protocol = 'ostatus'
        proto, _created = Protocol.objects.get_or_create(name=protocol)
        protocols.add(proto)
    node.protocols.set(protocols)
    services = set()
    for service in result.get('services', []):
        assert service != ""
        serv, _created = Service.objects.get_or_create(name=service)
        services.add(serv)
    node.services.set(services)

    activity = result.get('activity', {})
    users = activity.get('users', {})
    Stat.objects.update_or_create(
        node=node,
        date=now().date(),
        defaults={
            'users_total': users.get('total'),
            'users_half_year': users.get('half_year'),
            'users_monthly': users.get('monthly'),
            'users_weekly': users.get('weekly'),
            'local_posts': activity.get('local_posts'),
            'local_comments': activity.get('local_comments'),
        },
    )

    logger.info(f'Updated {host} successfully.')
    return True

def poll_nodes():
    logger.info(f'Queueing polling all nodes.')
    nodes_qs = Node.objects.only('host').active()
    for node in nodes_qs:
        poll_node.delay(node.host)
<|endoftext|>


<commit_before>#Python glue program to generate C options to be read from inifile

import StringIO

DEFAULT_OPTIONS_FILENAME = "default_options.ini"
C_HANDLER_SOURCE = "options.c"
C_OPTIONS_HEADER = "options.h"

#(Parameter_name,category,Parameter_type,Default_value)

scalar_options_set_1=[
("Omega_m","cosmo",float,0.26),
("Omega_l","cosmo",float,0.74),
("w","cosmo",float,-1.0),
("n_scalar","cosmo",float,0.960),
("sigma8","cosmo",float,0.798),
("z_source","cosmo",float,1.0)
]

scalar_options=[
("Cosmological parameters scalar",scalar_options_set_1)
]

#(Number_of_parameters_name,array_parameter_name,num_elements,Parameter_type,Default_values) (only numbers supported!!)
vector_options_set_1=[
("cosmovec","Nsi8","si8",3,float,0.798),
("cosmovec","Nh","h",4,float,0.7)
]

vector_options=[
("Cosmological parameters vector",vector_options_set_1)
]

def ini_string(value,ptype):
	if ptype==str:
		return value
	elif ptype in(float,int):
		return str(value)
	else:
		raise ValueError("Unknown parameter type")

def declaration_string(ptype):
	if ptype==str:
		return 'char*'
	elif ptype==int:
		return 'int'
	elif ptype==float:
		return 'float'
	else:
		raise ValueError("Unknown parameter type")

def print_format(ptype):
	if ptype==str:
		return '%s'
	elif ptype==int:
		return '%d'
	elif ptype==float:
		return '%f'
	else:
		raise ValueError("Unknown parameter type")

def declaration_match(ptype):
	if ptype==str:
		return 'strdup(value)'
	elif ptype==int:
		return 'atoi(value)'
	elif ptype==float:
		return 'atof(value)'
	else:
		raise ValueError("Unknown parameter type")

def generate_handler(scalar_options,vector_options):
	S=StringIO.StringIO()
	T=StringIO.StringIO()

	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "options.h"

int handler(void *user,const char *section,const char *name, const char *value){

		sys_options *options = (sys_options*) user;

		#define MATCH(s,n) strcmp(section,s)==0 && strcmp(name,n)==0
""")

	T.write("""\n\nvoid print_options(sys_options *options){
""")

	S.write("""\n
		if(0){
		}""")

	#Scalar options
	for set_name,options_set in scalar_options:

		T.write("""    printf("\\n[%s]\\n\\n");\n"""%options_set[0][1])

		for name,set,ptype,default in options_set:
			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,name,name,declaration_match(ptype)))
			T.write("""    printf("%s = %s\\n",options->%s);\n"""%(name,print_format(ptype),name))

	#Vector options
	for set_name,options_set in vector_options:

		T.write("""    printf("\\n[%s]\\n\\n");\n"""%options_set[0][0])

		for set,num_name,arr_name,num_elements,ptype,default in options_set:

			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,num_name,num_name,declaration_match(ptype)))
			T.write("""    printf("%s = %s\\n",options->%s);\n"""%(num_name,print_format(int),num_name))

			for i in range(num_elements):
				S.write(""" else if(MATCH("%s","%s[%d]")){
				options->%s[%d] = %s;
				}"""%(set,arr_name,i,arr_name,i,declaration_match(ptype)))
				T.write("""    printf("%s[%d] = %s\\n",options->%s[%d]);\n"""%(arr_name,i,print_format(ptype),arr_name,i))

	S.write(""" else{
		return 0;
		}
		return 1;
}""")

	T.write("""}""")

	S.seek(0)
	T.seek(0)
	return S.read(),T.read()

def generate_options_type(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#ifndef __OPTIONS_H
#define __OPTIONS_H

typedef struct{

""")

	for set_name,options_set in scalar_options:
		for name,set,ptype,default in options_set:
			S.write("""%s %s;\n"""%(declaration_string(ptype),name))

	for set_name,options_set in vector_options:
		for set,num_name,arr_name,num_elements,ptype,default in options_set:
			S.write("""int %s;\n"""%num_name)
			S.write("""%s %s[%d];\n"""%(declaration_string(ptype),arr_name,num_elements))

	S.write("""
} sys_options;

int handler(void *,const char*,const char*,const char*);
void print_options(sys_options *options);

#endif

""")

	S.seek(0)
	return S.read()

def generate_default_parameter_file(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""# Default options ini file generated from build_options.py #\n""")

#scalar options
	for set_name,options_set in scalar_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][1]))
		for name,set,ptype,default in options_set:
			S.write("""%s = %s\n""" %(name,ini_string(default,ptype)))

		S.write("""\n""")

#Vector options
	for set_name,options_set in vector_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][0]))
		for set,num_name,arr_name,num_elements,ptype,default in options_set:
			S.write("""%s = %s\n""" %(num_name,ini_string(num_elements,int)))
			for i in range(num_elements):
				S.write("""%s[%d] = %s\n"""%(arr_name,i,ini_string(default,ptype)))

		S.write("""\n""")

	S.seek(0)
	return S.read()

if(__name__=="__main__"):
	to_print = generate_handler(scalar_options,vector_options)
	file(C_HANDLER_SOURCE,"w").write(to_print[0])
	file(C_HANDLER_SOURCE,"a").write(to_print[1])

	file(C_OPTIONS_HEADER,"w").write(generate_options_type(scalar_options,vector_options))

	file(DEFAULT_OPTIONS_FILENAME,"w").write(generate_default_parameter_file(scalar_options,vector_options))
<commit_msg>Added comments to build_options.py<commit_after>#Python glue program to generate C options to be read from inifile

import StringIO

############################################
#########Type in your options here!!########
############################################

DEFAULT_OPTIONS_FILENAME = "default_options.ini"
C_HANDLER_SOURCE = "options.c"
C_OPTIONS_HEADER = "options.h"

#Scalar options (name = value)

#Format is: (Category,Parameter_name,Parameter_type,Default_value)

scalar_options_set_1=[
("cosmo","Omega_m",float,0.26),
("cosmo","Omega_l",float,0.74),
("cosmo","w",float,-1.0),
("cosmo","n_scalar",float,0.960),
("cosmo","sigma8",float,0.798),
("cosmo","z_source",float,1.0)
]

scalar_options=[
("Cosmological parameters scalar",scalar_options_set_1)
]

#Vector options (num_par = Size_of_the_vector; arr_name = Name_of_the_vector)

#Format is (Category,Number_of_parameters_name,array_parameter_name,num_elements,Parameter_type,Default_values) (only int,float supported)

vector_options_set_1=[
("cosmovec","Nsi8","si8",3,float,0.798),
("cosmovec","Nh","h",4,float,0.7)
]

vector_options=[
("Cosmological parameters vector",vector_options_set_1)
]

def ini_string(value,ptype):
	if ptype==str:
		return value
	elif ptype in(float,int):
		return str(value)
	else:
		raise ValueError("Unknown parameter type")

def declaration_string(ptype):
	if ptype==str:
		return 'char*'
	elif ptype==int:
		return 'int'
	elif ptype==float:
		return 'float'
	else:
		raise ValueError("Unknown parameter type")

def print_format(ptype):
	if ptype==str:
		return '%s'
	elif ptype==int:
		return '%d'
	elif ptype==float:
		return '%f'
	else:
		raise ValueError("Unknown parameter type")

def declaration_match(ptype):
	if ptype==str:
		return 'strdup(value)'
	elif ptype==int:
		return 'atoi(value)'
	elif ptype==float:
		return 'atof(value)'
	else:
		raise ValueError("Unknown parameter type")

def generate_handler(scalar_options,vector_options):
	S=StringIO.StringIO()
	T=StringIO.StringIO()

	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "options.h"

int handler(void *user,const char *section,const char *name, const char *value){

		sys_options *options = (sys_options*) user;

		#define MATCH(s,n) strcmp(section,s)==0 && strcmp(name,n)==0
""")

	T.write("""\n\nvoid print_options(sys_options *options){
""")

	S.write("""\n
		if(0){
		}""")

	#Scalar options
	for set_name,options_set in scalar_options:

		T.write("""    printf("\\n[%s]\\n\\n");\n"""%options_set[0][0])

		for set,name,ptype,default in options_set:
			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,name,name,declaration_match(ptype)))
			T.write("""    printf("%s = %s\\n",options->%s);\n"""%(name,print_format(ptype),name))

	#Vector options
	for set_name,options_set in vector_options:

		T.write("""    printf("\\n[%s]\\n\\n");\n"""%options_set[0][0])

		for set,num_name,arr_name,num_elements,ptype,default in options_set:

			S.write(""" else if(MATCH("%s","%s")){
				options->%s = %s;
				}"""%(set,num_name,num_name,declaration_match(ptype)))
			T.write("""    printf("%s = %s\\n",options->%s);\n"""%(num_name,print_format(int),num_name))

			for i in range(num_elements):
				S.write(""" else if(MATCH("%s","%s[%d]")){
				options->%s[%d] = %s;
				}"""%(set,arr_name,i,arr_name,i,declaration_match(ptype)))
				T.write("""    printf("%s[%d] = %s\\n",options->%s[%d]);\n"""%(arr_name,i,print_format(ptype),arr_name,i))

	S.write(""" else{
		return 0;
		}
		return 1;
}""")

	T.write("""}""")

	S.seek(0)
	T.seek(0)
	return S.read(),T.read()

def generate_options_type(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""/* DO NOT EDIT THIS FILE - IT IS GENERATED FROM build_options.py. Edit that. */

#ifndef __OPTIONS_H
#define __OPTIONS_H

typedef struct{

""")

	for set_name,options_set in scalar_options:
		for set,name,ptype,default in options_set:
			S.write("""%s %s;\n"""%(declaration_string(ptype),name))

	for set_name,options_set in vector_options:
		for set,num_name,arr_name,num_elements,ptype,default in options_set:
			S.write("""int %s;\n"""%num_name)
			S.write("""%s %s[%d];\n"""%(declaration_string(ptype),arr_name,num_elements))

	S.write("""
} sys_options;

int handler(void *,const char*,const char*,const char*);
void print_options(sys_options *options);

#endif

""")

	S.seek(0)
	return S.read()

def generate_default_parameter_file(scalar_options,vector_options):
	S=StringIO.StringIO()
	S.write("""# Default options ini file generated from build_options.py #\n""")

#scalar options
	for set_name,options_set in scalar_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][0]))
		for set,name,ptype,default in options_set:
			S.write("""%s = %s\n""" %(name,ini_string(default,ptype)))

		S.write("""\n""")

#Vector options
	for set_name,options_set in vector_options:
		S.write("""
##########################
##########%s##############
##########################
[%s]

"""%(set_name,options_set[0][0]))
		for set,num_name,arr_name,num_elements,ptype,default in options_set:
			S.write("""%s = %s\n""" %(num_name,ini_string(num_elements,int)))
			for i in range(num_elements):
				S.write("""%s[%d] = %s\n"""%(arr_name,i,ini_string(default,ptype)))

		S.write("""\n""")

	S.seek(0)
	return S.read()

if(__name__=="__main__"):
	to_print = generate_handler(scalar_options,vector_options)
	file(C_HANDLER_SOURCE,"w").write(to_print[0])
	file(C_HANDLER_SOURCE,"a").write(to_print[1])

	file(C_OPTIONS_HEADER,"w").write(generate_options_type(scalar_options,vector_options))

	file(DEFAULT_OPTIONS_FILENAME,"w").write(generate_default_parameter_file(scalar_options,vector_options))
<|endoftext|>


<commit_before># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid

import anyjson
import eventlet.debug
from oslo import messaging
from oslo.messaging import target

from murano.common import config
from murano.common.helpers import token_sanitizer
from murano.common import rpc
from murano.dsl import executor
from murano.dsl import results_serializer
from murano.dsl import virtual_exceptions
from murano.engine import environment
from murano.engine import package_class_loader
from murano.engine import package_loader
from murano.engine.system import status_reporter
import murano.engine.system.system_objects as system_objects
from murano.openstack.common.gettextutils import _  # noqa
from murano.openstack.common import log as logging

RPC_SERVICE = None

LOG = logging.getLogger(__name__)

eventlet.debug.hub_exceptions(False)

class TaskProcessingEndpoint(object):
    @staticmethod
    def handle_task(context, task):
        s_task = token_sanitizer.TokenSanitizer().sanitize(task)
        LOG.info(_('Starting processing task: {task_desc}').format(
            task_desc=anyjson.dumps(s_task)))

        try:
            task_executor = TaskExecutor(task)
            result = task_executor.execute()
            rpc.api().process_result(result)
        except Exception as e:
            # TODO(gokrokve) report error here
            # TODO(slagun) code below needs complete rewrite and redesign
            LOG.exception("Error during task execution for tenant %s",
                          task['tenant_id'])
            if task['model']['Objects']:
                msg_env = Environment(task['model']['Objects']['?']['id'])
                reporter = status_reporter.StatusReporter()
                reporter.initialize(msg_env)
                reporter.report_error(msg_env, str(e))
            rpc.api().process_result(task['model'])

def _prepare_rpc_service(server_id):
    endpoints = [TaskProcessingEndpoint()]

    transport = messaging.get_transport(config.CONF)
    s_target = target.Target('murano', 'tasks', server=server_id)
    return messaging.get_rpc_server(transport, s_target, endpoints, 'eventlet')

def get_rpc_service():
    global RPC_SERVICE

    if RPC_SERVICE is None:
        RPC_SERVICE = _prepare_rpc_service(str(uuid.uuid4()))
    return RPC_SERVICE

class Environment:
    def __init__(self, object_id):
        self.object_id = object_id

class TaskExecutor(object):
    @property
    def action(self):
        return self._action

    @property
    def environment(self):
        return self._environment

    @property
    def model(self):
        return self._model

    def __init__(self, task):
        self._action = task.get('action')
        self._model = task['model']
        self._environment = environment.Environment()
        self._environment.token = task['token']
        self._environment.tenant_id = task['tenant_id']

    def execute(self):
        token, tenant_id = self.environment.token, self.environment.tenant_id
        with package_loader.ApiPackageLoader(token, tenant_id) as pkg_loader:
            class_loader = package_class_loader.PackageClassLoader(pkg_loader)
            system_objects.register(class_loader, pkg_loader)

            exc = executor.MuranoDslExecutor(class_loader, self.environment)
            obj = exc.load(self.model)

            try:
                # Skip execution of action in case of no action is provided.
                # Model will be just loaded, cleaned-up and unloaded.
                # Most of the time this is used for deletion of environments.
                if self.action:
                    self._invoke(exc)
            except Exception as e:
                if isinstance(e, virtual_exceptions.MuranoPlException):
                    LOG.error(e.format())
                else:
                    LOG.exception(e)
                reporter = status_reporter.StatusReporter()
                reporter.initialize(obj)
                reporter.report_error(obj, str(e))

            return results_serializer.serialize(obj, exc)

    def _invoke(self, mpl_executor):
        obj = mpl_executor.object_store.get(self.action['object_id'])
        method_name, args = self.action['method'], self.action['args']

        if obj is not None:
            obj.type.invoke(method_name, mpl_executor, obj, args)
<commit_msg>MuranoPlException was referenced from incorrect module<commit_after># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid

import anyjson
import eventlet.debug
from oslo import messaging
from oslo.messaging import target

from murano.common import config
from murano.common.helpers import token_sanitizer
from murano.common import rpc
from murano.dsl import dsl_exception
from murano.dsl import executor
from murano.dsl import results_serializer
from murano.engine import environment
from murano.engine import package_class_loader
from murano.engine import package_loader
from murano.engine.system import status_reporter
import murano.engine.system.system_objects as system_objects
from murano.openstack.common.gettextutils import _  # noqa
from murano.openstack.common import log as logging

RPC_SERVICE = None

LOG = logging.getLogger(__name__)

eventlet.debug.hub_exceptions(False)

class TaskProcessingEndpoint(object):
    @staticmethod
    def handle_task(context, task):
        s_task = token_sanitizer.TokenSanitizer().sanitize(task)
        LOG.info(_('Starting processing task: {task_desc}').format(
            task_desc=anyjson.dumps(s_task)))

        try:
            task_executor = TaskExecutor(task)
            result = task_executor.execute()
            rpc.api().process_result(result)
        except Exception as e:
            # TODO(gokrokve) report error here
            # TODO(slagun) code below needs complete rewrite and redesign
            LOG.exception("Error during task execution for tenant %s",
                          task['tenant_id'])
            if task['model']['Objects']:
                msg_env = Environment(task['model']['Objects']['?']['id'])
                reporter = status_reporter.StatusReporter()
                reporter.initialize(msg_env)
                reporter.report_error(msg_env, str(e))
            rpc.api().process_result(task['model'])

def _prepare_rpc_service(server_id):
    endpoints = [TaskProcessingEndpoint()]

    transport = messaging.get_transport(config.CONF)
    s_target = target.Target('murano', 'tasks', server=server_id)
    return messaging.get_rpc_server(transport, s_target, endpoints, 'eventlet')

def get_rpc_service():
    global RPC_SERVICE

    if RPC_SERVICE is None:
        RPC_SERVICE = _prepare_rpc_service(str(uuid.uuid4()))
    return RPC_SERVICE

class Environment:
    def __init__(self, object_id):
        self.object_id = object_id

class TaskExecutor(object):
    @property
    def action(self):
        return self._action

    @property
    def environment(self):
        return self._environment

    @property
    def model(self):
        return self._model

    def __init__(self, task):
        self._action = task.get('action')
        self._model = task['model']
        self._environment = environment.Environment()
        self._environment.token = task['token']
        self._environment.tenant_id = task['tenant_id']

    def execute(self):
        token, tenant_id = self.environment.token, self.environment.tenant_id
        with package_loader.ApiPackageLoader(token, tenant_id) as pkg_loader:
            class_loader = package_class_loader.PackageClassLoader(pkg_loader)
            system_objects.register(class_loader, pkg_loader)

            exc = executor.MuranoDslExecutor(class_loader, self.environment)
            obj = exc.load(self.model)

            try:
                # Skip execution of action in case of no action is provided.
                # Model will be just loaded, cleaned-up and unloaded.
                # Most of the time this is used for deletion of environments.
                if self.action:
                    self._invoke(exc)
            except Exception as e:
                if isinstance(e, dsl_exception.MuranoPlException):
                    LOG.error(e.format())
                else:
                    LOG.exception(e)
                reporter = status_reporter.StatusReporter()
                reporter.initialize(obj)
                reporter.report_error(obj, str(e))

            return results_serializer.serialize(obj, exc)

    def _invoke(self, mpl_executor):
        obj = mpl_executor.object_store.get(self.action['object_id'])
        method_name, args = self.action['method'], self.action['args']

        if obj is not None:
            obj.type.invoke(method_name, mpl_executor, obj, args)
<|endoftext|>


<commit_before>import urllib
import urllib2
import urlparse
import logging
import re

log = logging.getLogger("newtorrents")

# this way we don't force users to install bs incase they do not want to use module
soup_present = True
soup_err = "Module newtorrents requires BeautifulSoup. Please install it from http://www.crummy.com/software/BeautifulSoup/ or from your distribution repository."

try:
    from BeautifulSoup import BeautifulSoup
except:
    log.warning(soup_err)
    soup_present = False

class ResolveNewTorrents:
    """NewTorrents resolver."""

    # a bit messy since this was not originally a resolver

    def __init__(self):
        self.resolved = []

    def register(self, manager, parser):
        manager.register_resolver(instance=self, name='newtorrents')

    def resolvable(self, feed, entry):
        # Return true only for urls that can and should be resolved
        return entry['url'].startswith('http://www.newtorrents.info') and not entry['url'] in self.resolved

    def resolve(self, feed, entry):
        if not soup_present:
            log.error(soup_err)
            return

        # resolve entry url
        url = entry['url']
        if url.startswith("http://www.newtorrents.info/?q=") or url.startswith("http://www.newtorrents.info/search"):
            url = self.__get_torrent_url_from_search(url, entry['title'])
        else:
            url = self.__get_torrent_url_from_page(url)

        if url:
            entry['url'] = url
            self.resolved.append(url)
            return True
        else:
            return False

    def __get_torrent_url_from_page(self, url):
        """Parses torrent url from newtorrents download page"""
        page = urllib2.urlopen(url)
        data = page.read()
        p = re.compile("copy\(\'(.*)\'\)", re.IGNORECASE)
        f = p.search(data)
        if f==None:
            log.error('Failed to get url from download page. Module may need a update.')
            return None
        else:
            return f.groups()[0]

    def __get_torrent_url_from_search(self, url, name):
        """Parses torrent download url (requires release name) from search results"""
        name = name.replace('.',' ').lower()
        try:
            page = urllib2.urlopen(url)
        except URLError, e:
            log.warning("Timed out when opening page")
            return

        soup = BeautifulSoup(page)
        torrents = []
        for link in soup.findAll('a', attrs={'href': re.compile('down.php')}):
            torrent_url = "http://www.newtorrents.info%s" % link.get('href')
            release_name = link.parent.next.get('title').replace('.',' ').lower()
            if release_name == name:
                torrents.append(torrent_url)
            else:
                log.debug("rejecting search result: '%s' != '%s'" % (release_name, name))

        # choose the torrent
        if not torrents:
            log.debug("did not find any matches in search result")
            return None
        else:
            if len(torrents) == 1:
                log.debug("found only one matching search result.")
            else:
                log.debug('search result contains multiple matches, using first occurence from: %s' % torrents)
                # TODO: use the one that has most downloaders / seeders
            return torrents[0]

if __name__ == '__main__':
    import sys
    logging.basicConfig(level=logging.DEBUG)
    import test_tools
    test_tools.test_resolver(ResolveNewTorrents())
<commit_msg>timeout check added<commit_after>import urllib
import urllib2
import urlparse
import logging
import re

log = logging.getLogger("newtorrents")

# this way we don't force users to install bs incase they do not want to use module
soup_present = True
soup_err = "Module newtorrents requires BeautifulSoup. Please install it from http://www.crummy.com/software/BeautifulSoup/ or from your distribution repository."

try:
    from BeautifulSoup import BeautifulSoup
except:
    log.warning(soup_err)
    soup_present = False

class ResolveNewTorrents:
    """NewTorrents resolver."""

    # a bit messy since this was not originally a resolver

    def __init__(self):
        self.resolved = []

    def register(self, manager, parser):
        manager.register_resolver(instance=self, name='newtorrents')

    def resolvable(self, feed, entry):
        # Return true only for urls that can and should be resolved
        return entry['url'].startswith('http://www.newtorrents.info') and not entry['url'] in self.resolved

    def resolve(self, feed, entry):
        if not soup_present:
            log.error(soup_err)
            return

        # resolve entry url
        url = entry['url']
        if url.startswith("http://www.newtorrents.info/?q=") or url.startswith("http://www.newtorrents.info/search"):
            url = self.__get_torrent_url_from_search(url, entry['title'])
        else:
            url = self.__get_torrent_url_from_page(url)

        if url:
            entry['url'] = url
            self.resolved.append(url)
            return True
        else:
            return False

    def __get_torrent_url_from_page(self, url):
        """Parses torrent url from newtorrents download page"""
        try:
            page = urllib2.urlopen(url)
            data = page.read()
        except urllib2.URLError, e:
            log.error("URLerror when retrieving page")
            return None
        p = re.compile("copy\(\'(.*)\'\)", re.IGNORECASE)
        f = p.search(data)
        if f==None:
            log.error('Failed to get url from download page. Module may need a update.')
            return None
        else:
            return f.groups()[0]

    def __get_torrent_url_from_search(self, url, name):
        """Parses torrent download url (requires release name) from search results"""
        name = name.replace('.',' ').lower()
        try:
            page = urllib2.urlopen(url)
        except URLError, e:
            log.warning("Timed out when opening page")
            return

        soup = BeautifulSoup(page)
        torrents = []
        for link in soup.findAll('a', attrs={'href': re.compile('down.php')}):
            torrent_url = "http://www.newtorrents.info%s" % link.get('href')
            release_name = link.parent.next.get('title').replace('.',' ').lower()
            if release_name == name:
                torrents.append(torrent_url)
            else:
                log.debug("rejecting search result: '%s' != '%s'" % (release_name, name))

        # choose the torrent
        if not torrents:
            log.debug("did not find any matches in search result")
            return None
        else:
            if len(torrents) == 1:
                log.debug("found only one matching search result.")
            else:
                log.debug('search result contains multiple matches, using first occurence from: %s' % torrents)
                # TODO: use the one that has most downloaders / seeders
            return torrents[0]

if __name__ == '__main__':
    import sys
    logging.basicConfig(level=logging.DEBUG)
    import test_tools
    test_tools.test_resolver(ResolveNewTorrents())
<|endoftext|>


<commit_before>
import json

import apps.archive  # NOQA
# XXX: This import is needed in order to avoid ImportError when importing
# package_service, caused by circular dependencies.
# When that issue is resolved, the workaround should be removed.

import apps.packages.package_service as package

from superdesk import get_resource_service
from superdesk.services import BaseService
from eve.utils import ParsedRequest
from superdesk.notification import push_notification

def init_parsed_request(elastic_query):
    parsed_request = ParsedRequest()
    parsed_request.args = {"source": json.dumps(elastic_query)}
    return parsed_request

def get_highlighted_items(highlights_id):
    """Get items marked for given highlight and passing date range query."""
    highlight = get_resource_service('highlights').find_one(req=None, _id=highlights_id)
    query = {
        'query': {
            'filtered': {'filter': {'and': [
                {'range': {'versioncreated': {'gte': highlight.get('auto_insert', 'now/d')}}},
                {'term': {'highlights': str(highlights_id)}},
            ]}}
        },
        'sort': [
            {'versioncreated': 'desc'},
        ]
    }
    request = ParsedRequest()
    request.args = {'source': json.dumps(query), 'repo': 'archive,published'}
    return list(get_resource_service('search').get(req=request, lookup=None))

def init_highlight_package(doc):
    """Add to package items marked for doc highlight."""
    main_group = doc.get('groups')[1]
    items = get_highlighted_items(doc.get('highlight'))
    for item in items:
        main_group['refs'].append(package.get_item_ref(item))

def on_create_package(sender, docs):
    """Call init_highlight_package for each package with highlight reference."""
    for doc in docs:
        if doc.get('highlight'):
            init_highlight_package(doc)

class HighlightsService(BaseService):
    def on_delete(self, doc):
        service = get_resource_service('archive')
        highlights_id = str(doc['_id'])
        query = {'query': {'filtered': {'filter': {'term': {'highlights': highlights_id}}}}}
        req = init_parsed_request(query)
        proposedItems = service.get(req=req, lookup=None)
        for item in proposedItems:
            updates = item.get('highlights').remove(highlights_id)
            service.update(item['_id'], {'highlights': updates}, item)

class MarkedForHighlightsService(BaseService):
    def create(self, docs, **kwargs):
        """Toggle highlight status for given highlight and item."""
        service = get_resource_service('archive')
        publishedService = get_resource_service('published')
        ids = []
        for doc in docs:
            item = service.find_one(req=None, guid=doc['marked_item'])
            if not item:
                ids.append(None)
                continue
            ids.append(item['_id'])
            highlights = item.get('highlights', [])
            if not highlights:
                highlights = []

            if doc['highlights'] not in highlights:
                highlights.append(doc['highlights'])
                highlight_on = True  # highlight toggled on
            else:
                highlights = [h for h in highlights if h != doc['highlights']]
                highlight_on = False  # highlight toggled off

            updates = {
                'highlights': highlights,
                '_updated': item['_updated'],
                '_etag': item['_etag']
            }
            service.update(item['_id'], updates, item)

            publishedItems = publishedService.find({'item_id': item['_id']})
            for publishedItem in publishedItems:
                updates = {
                    'highlights': highlights,
                    '_updated': publishedItem['_updated'],
                    '_etag': publishedItem['_etag']
                }
                publishedService.update(publishedItem['_id'], updates, publishedItem)

            push_notification(
                'item:highlight',
                marked=int(highlight_on),
                item_id=item['_id'],
                highlight_id=str(doc['highlights']))

        return ids

package.package_create_signal.connect(on_create_package)
<commit_msg>[SD-4439] - Avoid duplicated items in highlight creation<commit_after>
import json

import apps.archive  # NOQA
# XXX: This import is needed in order to avoid ImportError when importing
# package_service, caused by circular dependencies.
# When that issue is resolved, the workaround should be removed.

import apps.packages.package_service as package

from superdesk import get_resource_service
from superdesk.services import BaseService
from eve.utils import ParsedRequest
from superdesk.notification import push_notification

def init_parsed_request(elastic_query):
    parsed_request = ParsedRequest()
    parsed_request.args = {"source": json.dumps(elastic_query)}
    return parsed_request

def get_highlighted_items(highlights_id):
    """Get items marked for given highlight and passing date range query."""
    highlight = get_resource_service('highlights').find_one(req=None, _id=highlights_id)
    query = {
        'query': {
            'filtered': {'filter': {'and': [
                {'range': {'versioncreated': {'gte': highlight.get('auto_insert', 'now/d')}}},
                {'term': {'highlights': str(highlights_id)}},
            ]}}
        },
        'sort': [
            {'versioncreated': 'desc'},
        ]
    }
    request = ParsedRequest()
    request.args = {'source': json.dumps(query), 'repo': 'archive,published'}
    return list(get_resource_service('search').get(req=request, lookup=None))

def init_highlight_package(doc):
    """Add to package items marked for doc highlight."""
    main_group = doc.get('groups')[1]
    items = get_highlighted_items(doc.get('highlight'))
    used_items = []
    for item in items:
        if item['_id'] not in used_items:
            main_group['refs'].append(package.get_item_ref(item))
            used_items.append(item['_id'])

def on_create_package(sender, docs):
    """Call init_highlight_package for each package with highlight reference."""
    for doc in docs:
        if doc.get('highlight'):
            init_highlight_package(doc)

class HighlightsService(BaseService):
    def on_delete(self, doc):
        service = get_resource_service('archive')
        highlights_id = str(doc['_id'])
        query = {'query': {'filtered': {'filter': {'term': {'highlights': highlights_id}}}}}
        req = init_parsed_request(query)
        proposedItems = service.get(req=req, lookup=None)
        for item in proposedItems:
            updates = item.get('highlights').remove(highlights_id)
            service.update(item['_id'], {'highlights': updates}, item)

class MarkedForHighlightsService(BaseService):
    def create(self, docs, **kwargs):
        """Toggle highlight status for given highlight and item."""
        service = get_resource_service('archive')
        publishedService = get_resource_service('published')
        ids = []
        for doc in docs:
            item = service.find_one(req=None, guid=doc['marked_item'])
            if not item:
                ids.append(None)
                continue
            ids.append(item['_id'])
            highlights = item.get('highlights', [])
            if not highlights:
                highlights = []

            if doc['highlights'] not in highlights:
                highlights.append(doc['highlights'])
                highlight_on = True  # highlight toggled on
            else:
                highlights = [h for h in highlights if h != doc['highlights']]
                highlight_on = False  # highlight toggled off

            updates = {
                'highlights': highlights,
                '_updated': item['_updated'],
                '_etag': item['_etag']
            }
            service.update(item['_id'], updates, item)

            publishedItems = publishedService.find({'item_id': item['_id']})
            for publishedItem in publishedItems:
                if publishedItem['_current_version'] == item['_current_version'] or not highlight_on:
                    updates = {
                        'highlights': highlights,
                        '_updated': publishedItem['_updated'],
                        '_etag': publishedItem['_etag']
                    }
                    publishedService.update(publishedItem['_id'], updates, publishedItem)

            push_notification(
                'item:highlight',
                marked=int(highlight_on),
                item_id=item['_id'],
                highlight_id=str(doc['highlights']))

        return ids

package.package_create_signal.connect(on_create_package)
<|endoftext|>


<commit_before>"""
The Message class represents a message that is sent or received and contains
methods for publishing the message, or in the case that the message was
delivered by RabbitMQ, acknowledging it, rejecting it or negatively
acknowledging it.

"""
import datetime
import json
import logging
import math
import time
import pprint
import uuid

from pamqp import body
from pamqp import header
from pamqp import specification

from rabbitpy import base
from rabbitpy import exceptions
from rabbitpy import utils

LOGGER = logging.getLogger(__name__)

class Properties(specification.Basic.Properties):
    """Proxy class for :py:class:`pamqp.specification.Basic.Properties`"""
    pass

class Message(base.AMQPClass):
    """Created by both rabbitpy internally when a message is delivered or
    returned from RabbitMQ and by implementing applications, the Message class
    is used to publish a message to and access and respond to a message from
    RabbitMQ.

    When specifying properties for a message, pass in a dict of key value items
    that match the AMQP Basic.Properties specification with a small caveat.

    Due to an overlap in the AMQP specification and the Python keyword
    :code:`type`, the :code:`type` property is referred to as
    :code:`message_type`.

    The following is a list of the available properties:

    * app_id
    * content_type
    * content_encoding
    * correlation_id
    * delivery_node
    * expiration
    * headers
    * message_id
    * message_type
    * priority
    * reply_to
    * timestamp
    * user_id

    **Automated features**

    When passing in the body value, if it is a dict or list, it will
    automatically be JSON serialized and the content type ``application/json``
    will be set on the message properties.

    When publishing a message to RabbitMQ, if the opinionated value is ``True``
    and no ``message_id`` value was passed in as a property, a UUID will be
    generated and specified as a property of the message.

    Additionally, if opinionated is ``True`` and the ``timestamp`` property
    is not specified when passing in ``properties``, the current Unix epoch
    value will be set in the message properties.

    .. note:: As of 0.20.0 ``auto_id`` is deprecated in favor of
    ``opinionated`` and it will be removed in 0.21.0. Additionally for
    compatibility, ``0.20.0`` has ``opinionated`` defaulted to ``True``. This
    will change to ``False`` in 0.21.0.

    :param channel: The channel object for the message object to act upon
    :type channel: :py:class:`rabbitpy.channel.Channel`
    :param str or dict or list body_value: The message body
    :param dict properties: A dictionary of message properties
    :param bool auto_id: Add a message id if no properties were passed in.
    :param bool opinionated: Automatically populate properties if True
    :raises KeyError: Raised when an invalid property is passed in

    """
    method = None
    name = 'Message'

    def __init__(self, channel, body_value, properties=None,
                 auto_id=False, opinionated=True):
        """Create a new instance of the Message object."""
        super(Message, self).__init__(channel, 'Message')

        # Always have a dict of properties set
        self.properties = properties or {}

        # Assign the body value
        self.body = self._auto_serialize(body_value)

        # Add a message id if auto_id is not turned off and it is not set
        if (opinionated or auto_id) and 'message_id' not in self.properties:
            if auto_id:
                raise DeprecationWarning('Use opinionated instead of auto_id')
            self._add_auto_message_id()

        if opinionated:
            if 'timestamp' not in self.properties:
                self._add_timestamp()

        # Enforce datetime timestamps
        if 'timestamp' in self.properties:
            self.properties['timestamp'] = \
                self._as_datetime(self.properties['timestamp'])

        # Don't let invalid property keys in
        if self._invalid_properties:
            msg = 'Invalid property: %s' % self._invalid_properties[0]
            raise KeyError(msg)

    @property
    def delivery_tag(self):
        """Return the delivery tag for a message that was delivered or gotten
        from RabbitMQ.

        :rtype: int or None

        """
        return self.method.delivery_tag if self.method else None

    @property
    def redelivered(self):
        """Indicates if this message may have been delivered before (but not
        acknowledged)"

        :rtype: bool or None

        """
        return self.method.redelivered if self.method else None

    @property
    def routing_key(self):
        """Return the routing_key for a message that was delivered or gotten
        from RabbitMQ.

        :rtype: int or None

        """
        return self.method.routing_key if self.method else None

    @property
    def exchange(self):
        """Return the source exchange for a message that was delivered or
        gotten from RabbitMQ.

        :rtype: string or None

        """
        return self.method.exchange if self.method else None

    def ack(self, all_previous=False):
        """Acknowledge receipt of the message to RabbitMQ. Will raise an
        ActionException if the message was not received from a broker.

        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not ack non-received '
                                             'message')
        basic_ack = specification.Basic.Ack(self.method.delivery_tag,
                                            multiple=all_previous)
        self.channel._write_frame(basic_ack)

    def json(self):
        """Deserialize the message body if it is JSON, returning the value.

        :rtype: any

        """
        try:
            return json.loads(self.body)
        except TypeError:  # pragma: no cover
            return json.loads(self.body.decode('utf-8'))

    def nack(self, requeue=False, all_previous=False):
        """Negatively acknowledge receipt of the message to RabbitMQ. Will
        raise an ActionException if the message was not received from a broker.

        :param bool requeue: Requeue the message
        :param bool all_previous: Nack all previous unacked messages up to and
                                  including this one
        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not nack non-received '
                                             'message')
        basic_nack = specification.Basic.Nack(self.method.delivery_tag,
                                              requeue=requeue,
                                              multiple=all_previous)
        self.channel._write_frame(basic_nack)

    def pprint(self, properties=False):  # pragma: no cover
        """Print a formatted representation of the message.

        :param bool properties: Include properties in the representation

        """
        print('Exchange: %s\n' % self.method.exchange)
        print('Routing Key: %s\n' % self.method.routing_key)
        if properties:
            print('Properties:\n')
            pprint.pprint(self.properties)
            print('\nBody:\n')
        pprint.pprint(self.body)

    def publish(self, exchange, routing_key='', mandatory=False):
        """Publish the message to the exchange with the specified routing
        key.

        In Python 2 if the message is a ``unicode`` value it will be converted
        to a ``str`` using ``str.encode('UTF-8')``. If you do not want the
        auto-conversion to take place, set the body to a ``str`` or ``bytes``
        value prior to publishing.

        In Python 3 if the message is a ``str`` value it will be converted to
        a ``bytes`` value using ``bytes(value.encode('UTF-8'))``. If you do
        not want the auto-conversion to take place, set the body to a
        ``bytes`` value prior to publishing.

        :param exchange: The exchange to publish the message to
        :type exchange: str or :class:`rabbitpy.Exchange`
        :param str routing_key: The routing key to use
        :param bool mandatory: Requires the message is published
        :return: bool or None
        :raises: rabbitpy.exceptions.MessageReturnedException

        """
        if isinstance(exchange, base.AMQPClass):
            exchange = exchange.name

        frames = [specification.Basic.Publish(exchange=exchange,
                                              routing_key=routing_key or '',
                                              mandatory=mandatory),
                  header.ContentHeader(body_size=len(self.body),
                                       properties=self._properties)]

        # Coerce the body to the proper type
        self._coerce_body()

        # Calculate how many body frames are needed
        pieces = int(math.ceil(len(self.body) /
                               float(self.channel.maximum_frame_size)))

        # Send the message
        for offset in range(0, pieces):
            start = self.channel.maximum_frame_size * offset
            end = start + self.channel.maximum_frame_size
            if end > len(self.body):
                end = len(self.body)
            frames.append(body.ContentBody(self.body[start:end]))

        # Write the frames out
        self.channel._write_frames(frames)

        # If publisher confirmations are enabled, wait for the response
        if self.channel.publisher_confirms:
            response = self.channel._wait_for_confirmation()
            if isinstance(response, specification.Basic.Ack):
                return True
            elif isinstance(response, specification.Basic.Nack):
                return False
            else:
                raise exceptions.UnexpectedResponseError(response)

    def reject(self, requeue=False):
        """Reject receipt of the message to RabbitMQ. Will raise
        an ActionException if the message was not received from a broker.

        :param bool requeue: Requeue the message
        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not reject non-received '
                                             'message')
        basic_reject = specification.Basic.Reject(self.method.delivery_tag,
                                                  requeue=requeue)
        self.channel._write_frame(basic_reject)

    def _add_auto_message_id(self):
        """Set the message_id property to a new UUID."""
        self.properties['message_id'] = str(uuid.uuid4())

    def _add_timestamp(self):
        """Add the timestamp to the properties"""
        self.properties['timestamp'] = datetime.datetime.now()

    def _as_datetime(self, value):
        """Return the passed in value as a ``datetime.datetime`` value.

        :param value: The value to convert or pass through
        :type value: datetime.datetime
        :type value: time.struct_time
        :type value: int
        :type value: float
        :type value: str
        :type value: bytes
        :type value: unicode
        :rtype: datetime.datetime
        :raises: ValueError

        """
        if value is None:
            return None

        if isinstance(value, datetime.datetime):
            return value

        if isinstance(value, time.struct_time):
            return datetime.datetime(*value[:6])

        if utils.is_string(value):
            value = int(value)

        if isinstance(value, float) or isinstance(value, int):
            return datetime.datetime.fromtimestamp(value)

        raise ValueError('Could not cast a %s value to a datetime.datetime' %
                         type(value))

    def _auto_serialize(self, body_value):
        """Automatically serialize the body as JSON if it is a dict or list.

        :param mixed body_value: The message body passed into the constructor
        :return: str or bytes or unicode or None

        """
        if isinstance(body_value, dict) or isinstance(body_value, list):
            self.properties['content_type'] = 'application/json'
            return json.dumps(body_value, ensure_ascii=False)
        return body_value

    def _coerce_body(self): # pragma: no cover
        """Ensure that unicode body values are cast to strings or bytes.

        """
        try:
            if isinstance(self.body, unicode):
                self.body = self.body.encode('UTF-8')
        except NameError:
            if isinstance(self.body, str):
                self.body = bytes(self.body.encode('UTF-8'))

    def _coerce_properties(self):
        """Force properties to be set to the correct data type"""
        for key in self.properties:
            _type = getattr(specification.Basic.Properties, key)
            if _type == 'shortstr' and \
                    not utils.is_string(self.properties[key]):
                LOGGER.warning('Coercing property %s to bytes', key)
                self.properties[key] = bytes(self.properties[key])
            elif _type == 'octet' and not isinstance(self.properties[key],
                                                     int):
                LOGGER.warning('Coercing property %s to int', key)
                self.properties[key] = int(self.properties[key])
            elif _type == 'table' and not isinstance(self.properties[key],
                                                     dict):
                LOGGER.warning('Resetting invalid value for %s to None', key)
                self.properties[key] = {}
            if key == 'timestamp':
                self.properties[key] = self._as_datetime(self.properties[key])

    @property
    def _invalid_properties(self):
        """Return a list of invalid properties that currently exist in the the
        properties that are set.

        :rtype: list

        """
        return [key for key in self.properties
                if key not in specification.Basic.Properties.attributes]

    @property
    def _properties(self):
        """Return a new Basic.Properties object representing the message
        properties.

        :rtype: pamqp.specification.Basic.Properties

        """
        self._prune_invalid_properties()
        self._coerce_properties()
        return specification.Basic.Properties(**self.properties)

    def _prune_invalid_properties(self):
        """Remove invalid properties from the message properties."""
        for key in self._invalid_properties:
            LOGGER.warning('Removing invalid property "%s"', key)
            del self.properties[key]
<commit_msg>Update versions in the docstring notes on deprecation<commit_after>"""
The Message class represents a message that is sent or received and contains
methods for publishing the message, or in the case that the message was
delivered by RabbitMQ, acknowledging it, rejecting it or negatively
acknowledging it.

"""
import datetime
import json
import logging
import math
import time
import pprint
import uuid

from pamqp import body
from pamqp import header
from pamqp import specification

from rabbitpy import base
from rabbitpy import exceptions
from rabbitpy import utils

LOGGER = logging.getLogger(__name__)

class Properties(specification.Basic.Properties):
    """Proxy class for :py:class:`pamqp.specification.Basic.Properties`"""
    pass

class Message(base.AMQPClass):
    """Created by both rabbitpy internally when a message is delivered or
    returned from RabbitMQ and by implementing applications, the Message class
    is used to publish a message to and access and respond to a message from
    RabbitMQ.

    When specifying properties for a message, pass in a dict of key value items
    that match the AMQP Basic.Properties specification with a small caveat.

    Due to an overlap in the AMQP specification and the Python keyword
    :code:`type`, the :code:`type` property is referred to as
    :code:`message_type`.

    The following is a list of the available properties:

    * app_id
    * content_type
    * content_encoding
    * correlation_id
    * delivery_node
    * expiration
    * headers
    * message_id
    * message_type
    * priority
    * reply_to
    * timestamp
    * user_id

    **Automated features**

    When passing in the body value, if it is a dict or list, it will
    automatically be JSON serialized and the content type ``application/json``
    will be set on the message properties.

    When publishing a message to RabbitMQ, if the opinionated value is ``True``
    and no ``message_id`` value was passed in as a property, a UUID will be
    generated and specified as a property of the message.

    Additionally, if opinionated is ``True`` and the ``timestamp`` property
    is not specified when passing in ``properties``, the current Unix epoch
    value will be set in the message properties.

    .. note:: As of 0.21.0 ``auto_id`` is deprecated in favor of
    ``opinionated`` and it will be removed in 0.22.0. Additionally for
    compatibility, ``0.21.0`` has ``opinionated`` defaulted to ``True``. This
    will change to ``False`` in 0.22.0.

    :param channel: The channel object for the message object to act upon
    :type channel: :py:class:`rabbitpy.channel.Channel`
    :param str or dict or list body_value: The message body
    :param dict properties: A dictionary of message properties
    :param bool auto_id: Add a message id if no properties were passed in.
    :param bool opinionated: Automatically populate properties if True
    :raises KeyError: Raised when an invalid property is passed in

    """
    method = None
    name = 'Message'

    def __init__(self, channel, body_value, properties=None,
                 auto_id=False, opinionated=True):
        """Create a new instance of the Message object."""
        super(Message, self).__init__(channel, 'Message')

        # Always have a dict of properties set
        self.properties = properties or {}

        # Assign the body value
        self.body = self._auto_serialize(body_value)

        # Add a message id if auto_id is not turned off and it is not set
        if (opinionated or auto_id) and 'message_id' not in self.properties:
            if auto_id:
                raise DeprecationWarning('Use opinionated instead of auto_id')
            self._add_auto_message_id()

        if opinionated:
            if 'timestamp' not in self.properties:
                self._add_timestamp()

        # Enforce datetime timestamps
        if 'timestamp' in self.properties:
            self.properties['timestamp'] = \
                self._as_datetime(self.properties['timestamp'])

        # Don't let invalid property keys in
        if self._invalid_properties:
            msg = 'Invalid property: %s' % self._invalid_properties[0]
            raise KeyError(msg)

    @property
    def delivery_tag(self):
        """Return the delivery tag for a message that was delivered or gotten
        from RabbitMQ.

        :rtype: int or None

        """
        return self.method.delivery_tag if self.method else None

    @property
    def redelivered(self):
        """Indicates if this message may have been delivered before (but not
        acknowledged)"

        :rtype: bool or None

        """
        return self.method.redelivered if self.method else None

    @property
    def routing_key(self):
        """Return the routing_key for a message that was delivered or gotten
        from RabbitMQ.

        :rtype: int or None

        """
        return self.method.routing_key if self.method else None

    @property
    def exchange(self):
        """Return the source exchange for a message that was delivered or
        gotten from RabbitMQ.

        :rtype: string or None

        """
        return self.method.exchange if self.method else None

    def ack(self, all_previous=False):
        """Acknowledge receipt of the message to RabbitMQ. Will raise an
        ActionException if the message was not received from a broker.

        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not ack non-received '
                                             'message')
        basic_ack = specification.Basic.Ack(self.method.delivery_tag,
                                            multiple=all_previous)
        self.channel._write_frame(basic_ack)

    def json(self):
        """Deserialize the message body if it is JSON, returning the value.

        :rtype: any

        """
        try:
            return json.loads(self.body)
        except TypeError:  # pragma: no cover
            return json.loads(self.body.decode('utf-8'))

    def nack(self, requeue=False, all_previous=False):
        """Negatively acknowledge receipt of the message to RabbitMQ. Will
        raise an ActionException if the message was not received from a broker.

        :param bool requeue: Requeue the message
        :param bool all_previous: Nack all previous unacked messages up to and
                                  including this one
        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not nack non-received '
                                             'message')
        basic_nack = specification.Basic.Nack(self.method.delivery_tag,
                                              requeue=requeue,
                                              multiple=all_previous)
        self.channel._write_frame(basic_nack)

    def pprint(self, properties=False):  # pragma: no cover
        """Print a formatted representation of the message.

        :param bool properties: Include properties in the representation

        """
        print('Exchange: %s\n' % self.method.exchange)
        print('Routing Key: %s\n' % self.method.routing_key)
        if properties:
            print('Properties:\n')
            pprint.pprint(self.properties)
            print('\nBody:\n')
        pprint.pprint(self.body)

    def publish(self, exchange, routing_key='', mandatory=False):
        """Publish the message to the exchange with the specified routing
        key.

        In Python 2 if the message is a ``unicode`` value it will be converted
        to a ``str`` using ``str.encode('UTF-8')``. If you do not want the
        auto-conversion to take place, set the body to a ``str`` or ``bytes``
        value prior to publishing.

        In Python 3 if the message is a ``str`` value it will be converted to
        a ``bytes`` value using ``bytes(value.encode('UTF-8'))``. If you do
        not want the auto-conversion to take place, set the body to a
        ``bytes`` value prior to publishing.

        :param exchange: The exchange to publish the message to
        :type exchange: str or :class:`rabbitpy.Exchange`
        :param str routing_key: The routing key to use
        :param bool mandatory: Requires the message is published
        :return: bool or None
        :raises: rabbitpy.exceptions.MessageReturnedException

        """
        if isinstance(exchange, base.AMQPClass):
            exchange = exchange.name

        frames = [specification.Basic.Publish(exchange=exchange,
                                              routing_key=routing_key or '',
                                              mandatory=mandatory),
                  header.ContentHeader(body_size=len(self.body),
                                       properties=self._properties)]

        # Coerce the body to the proper type
        self._coerce_body()

        # Calculate how many body frames are needed
        pieces = int(math.ceil(len(self.body) /
                               float(self.channel.maximum_frame_size)))

        # Send the message
        for offset in range(0, pieces):
            start = self.channel.maximum_frame_size * offset
            end = start + self.channel.maximum_frame_size
            if end > len(self.body):
                end = len(self.body)
            frames.append(body.ContentBody(self.body[start:end]))

        # Write the frames out
        self.channel._write_frames(frames)

        # If publisher confirmations are enabled, wait for the response
        if self.channel.publisher_confirms:
            response = self.channel._wait_for_confirmation()
            if isinstance(response, specification.Basic.Ack):
                return True
            elif isinstance(response, specification.Basic.Nack):
                return False
            else:
                raise exceptions.UnexpectedResponseError(response)

    def reject(self, requeue=False):
        """Reject receipt of the message to RabbitMQ. Will raise
        an ActionException if the message was not received from a broker.

        :param bool requeue: Requeue the message
        :raises: ActionException

        """
        if not self.method:
            raise exceptions.ActionException('Can not reject non-received '
                                             'message')
        basic_reject = specification.Basic.Reject(self.method.delivery_tag,
                                                  requeue=requeue)
        self.channel._write_frame(basic_reject)

    def _add_auto_message_id(self):
        """Set the message_id property to a new UUID."""
        self.properties['message_id'] = str(uuid.uuid4())

    def _add_timestamp(self):
        """Add the timestamp to the properties"""
        self.properties['timestamp'] = datetime.datetime.now()

    def _as_datetime(self, value):
        """Return the passed in value as a ``datetime.datetime`` value.

        :param value: The value to convert or pass through
        :type value: datetime.datetime
        :type value: time.struct_time
        :type value: int
        :type value: float
        :type value: str
        :type value: bytes
        :type value: unicode
        :rtype: datetime.datetime
        :raises: ValueError

        """
        if value is None:
            return None

        if isinstance(value, datetime.datetime):
            return value

        if isinstance(value, time.struct_time):
            return datetime.datetime(*value[:6])

        if utils.is_string(value):
            value = int(value)

        if isinstance(value, float) or isinstance(value, int):
            return datetime.datetime.fromtimestamp(value)

        raise ValueError('Could not cast a %s value to a datetime.datetime' %
                         type(value))

    def _auto_serialize(self, body_value):
        """Automatically serialize the body as JSON if it is a dict or list.

        :param mixed body_value: The message body passed into the constructor
        :return: str or bytes or unicode or None

        """
        if isinstance(body_value, dict) or isinstance(body_value, list):
            self.properties['content_type'] = 'application/json'
            return json.dumps(body_value, ensure_ascii=False)
        return body_value

    def _coerce_body(self): # pragma: no cover
        """Ensure that unicode body values are cast to strings or bytes.

        """
        try:
            if isinstance(self.body, unicode):
                self.body = self.body.encode('UTF-8')
        except NameError:
            if isinstance(self.body, str):
                self.body = bytes(self.body.encode('UTF-8'))

    def _coerce_properties(self):
        """Force properties to be set to the correct data type"""
        for key in self.properties:
            _type = getattr(specification.Basic.Properties, key)
            if _type == 'shortstr' and \
                    not utils.is_string(self.properties[key]):
                LOGGER.warning('Coercing property %s to bytes', key)
                self.properties[key] = bytes(self.properties[key])
            elif _type == 'octet' and not isinstance(self.properties[key],
                                                     int):
                LOGGER.warning('Coercing property %s to int', key)
                self.properties[key] = int(self.properties[key])
            elif _type == 'table' and not isinstance(self.properties[key],
                                                     dict):
                LOGGER.warning('Resetting invalid value for %s to None', key)
                self.properties[key] = {}
            if key == 'timestamp':
                self.properties[key] = self._as_datetime(self.properties[key])

    @property
    def _invalid_properties(self):
        """Return a list of invalid properties that currently exist in the the
        properties that are set.

        :rtype: list

        """
        return [key for key in self.properties
                if key not in specification.Basic.Properties.attributes]

    @property
    def _properties(self):
        """Return a new Basic.Properties object representing the message
        properties.

        :rtype: pamqp.specification.Basic.Properties

        """
        self._prune_invalid_properties()
        self._coerce_properties()
        return specification.Basic.Properties(**self.properties)

    def _prune_invalid_properties(self):
        """Remove invalid properties from the message properties."""
        for key in self._invalid_properties:
            LOGGER.warning('Removing invalid property "%s"', key)
            del self.properties[key]
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from group import Group,run_grouping

def dist(tx, ty):
    return abs(tx - ty)

class Neighbor(object):
    def __init__(self, t, line):
        self.t = t
        self.line = line
        self.count = 0
        self.label = -1
        self.next = self.prev = None

    @classmethod
    def iterate_forward(cls, head):
        i = head
        while i != None:
            yield i
            i = i.next

    @classmethod
    def iterate_reverse(cls, tail):
        i = tail
        while i != None:
            yield i
            i = i.prev

    def neighbors(self):
        i = self.prev
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.prev
        i = self.next
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.next

class DBSCANGroup(Group):
    def __init__(self, tup):
        super(DBSCANGroup, self).__init__(tup)
        self.head = self.tail = None

    def add(self, chunks):
        n = Neighbor(findNumber(chunks[args.column]), args.jdelim.join(chunks))

        # Update neighbor counts
        oldest = None
        for nn in Neighbor.iterate_reverse(self.tail):
            if dist(nn.t, n.t) <= args.epsilon:
                n.count += 1
                nn.count += 1
                oldest = nn
            else:
                break

        # No neighbors for the new element, clear out old data
        if n.count == 0:
            self.empty()
        elif oldest.count >= args.min_samples: # Oldest is core
            if oldest.label != -1:
                label = oldest.label
            else:
                label = args.label
                args.label += 1

            for nn in oldest.neighbors():
                if nn.label == -1:
                    nn.label = label

        # Append onto the end of the list
        if self.tail:
            self.tail.next = n
        else:
            self.head = n
        n.prev = self.tail
        self.tail = n

    def empty(self):
        for nn in Neighbor.iterate_forward(self.head):
            args.outfile.write(nn.line + args.jdelim + str(nn.label) + '\n')
        self.head = self.tail = None

    def done(self):
        self.empty()

if __name__ == "__main__":
    # set up command line args
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,\
                                     description='Cluster input')
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-c', '--column', type=int, default=0)
    parser.add_argument('-e', '--epsilon', type=float, default=0.5)
    parser.add_argument('-m', '--min_samples', type=int, default=5)
    parser.add_argument('-g', '--group', nargs='+', type=int, default=[])
    parser.add_argument('-d', '--delimiter', default=None)
    parser.add_argument('-o', '--ordered', action='store_true', default=False, help='input is sorted by group')
    args = parser.parse_args()

    args.jdelim = args.delimiter if args.delimiter != None else ' '
    args.label = 0
    run_grouping(args.infile, DBSCANGroup, args.group, args.delimiter, args.ordered)

<commit_msg>fixed bug<commit_after>#!/usr/bin/python

import os
import sys
import argparse
import traceback
from input_handling import findNumber
from decimal import Decimal
from group import Group,run_grouping

def dist(tx, ty):
    return abs(tx - ty)

class Neighbor(object):
    def __init__(self, t, line):
        self.t = t
        self.line = line
        self.count = 0
        self.label = -1
        self.next = self.prev = None

    @classmethod
    def iterate_forward(cls, head):
        i = head
        while i != None:
            yield i
            i = i.next

    @classmethod
    def iterate_reverse(cls, tail):
        i = tail
        while i != None:
            yield i
            i = i.prev

    def neighbors(self):
        i = self.prev
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.prev
        i = self.next
        while i != None and dist(i.t, self.t) <= args.epsilon:
            yield i
            i = i.next

class DBSCANGroup(Group):
    def __init__(self, tup):
        super(DBSCANGroup, self).__init__(tup)
        self.head = self.tail = None

    def add(self, chunks):
        n = Neighbor(findNumber(chunks[args.column]), args.jdelim.join(chunks))

        # Update neighbor counts
        oldest = None
        for nn in Neighbor.iterate_reverse(self.tail):
            if dist(nn.t, n.t) <= args.epsilon:
                n.count += 1
                nn.count += 1
                oldest = nn
            else:
                break

        # No neighbors for the new element, clear out old data
        if n.count == 0:
            self.empty()
        elif oldest.count >= args.min_samples: # Oldest is core
            if oldest.label != -1:
                label = oldest.label
            else:
                label = args.label
                args.label += 1

            for nn in oldest.neighbors():
                if nn.label == -1:
                    nn.label = label

        # Append onto the end of the list
        if self.tail:
            self.tail.next = n
        else:
            self.head = n
        n.prev = self.tail
        self.tail = n

    def empty(self):
        for nn in Neighbor.iterate_forward(self.head):
            args.outfile.write(nn.line + args.jdelim + str(nn.label) + '\n')
        self.head = self.tail = None

    def done(self):
        self.empty()

if __name__ == "__main__":
    # set up command line args
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,\
                                     description='Cluster input')
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-c', '--column', type=int, default=0)
    parser.add_argument('-e', '--epsilon', type=Decimal, default=Decimal('0.5'))
    parser.add_argument('-m', '--min_samples', type=int, default=5)
    parser.add_argument('-g', '--group', nargs='+', type=int, default=[])
    parser.add_argument('-d', '--delimiter', default=None)
    parser.add_argument('-o', '--ordered', action='store_true', default=False, help='input is sorted by group')
    args = parser.parse_args()

    args.jdelim = args.delimiter if args.delimiter != None else ' '
    args.label = 0
    run_grouping(args.infile, DBSCANGroup, args.group, args.delimiter, args.ordered)

<|endoftext|>