<commit_before>from netaddr import IPNetwork, AddrFormatError
from elasticsearch import Elasticsearch
from elasticsearch import exceptions
from collections import OrderedDict
from pathlib import Path
import configparser
import argparse
import string
import json
import sys
import os
import re

def get_es_cluster_ip():
    """Returns the Elasticsearch IP from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['ELASTICSEARCH_IP']

def get_xpack_credentials():
    """Returns x-pack credentials from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['X-PACK_USERNAME'], config['elastic']['X-PACK_PASSWORD']

def xpack_enabled():
    """Returns whether x-pack is enabled from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['X-PACK_ENABLED']

def get_es_object():
    """Returns Elasticsearch object"""
    if xpack_enabled():
        credentials = get_xpack_credentials()
        return Elasticsearch([get_es_cluster_ip()], http_auth=(credentials[0], credentials[1]))
    else:
        return Elasticsearch(([{'host': get_es_cluster_ip()}]))

def es_get_all_ips(str_existing_index):
    """Returns list of list_of_ips stored in given Elasticsearch index"""
    list_ips = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"size": 0, "aggs": {"all_ip": {"terms": {"field": "ip", "size": count}}}})
    for key in res['aggregations']['all_ip']['buckets']:
        list_ips.append(key['key'])
    print('Found ' + str(len(list_ips)) + ' IPs in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return list_ips

def es_get_ips_by_query(str_existing_index):
    """Returns list of ips from query stored in given Elasticsearch index
    The query body in this function needs to be edited hardcoded.
    """
    list_ips = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"size": 0, "aggs": {"ips_by_query": {"terms": {"field": "ip", "size": count}}}, "query":
                        {"query_string": {"query":"\"cisco-IOS\" OR \" Cisco Systems\"", "analyze_wildcard": "true"}}})
    for key in res['aggregations']['ips_by_query']['buckets']:
        list_ips.append(key['key'])
    print('Found ' + str(len(list_ips)) + ' IPs by query in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return list_ips

def es_get_all(str_existing_index):
    """Returns all documents stored in given Elasticsearch index"""
    documents = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"query": { "match_all": {}}, "size": count})
    for key in res['hits']['hits']:
        documents.append(key)
    print('Found ' + str(len(documents)) + ' IPs in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return documents

def exists_es_index(str_valid_index):
    """Returns if given index exists in Elasticsearch cluster"""
    connection_attempts = 0
    while connection_attempts < 3:
        try:
            es = get_es_object()
            es_indices = es.indices
            return es_indices.exists(index=str_valid_index)
        except exceptions.ConnectionTimeout:
            connection_attempts += 1
    print('Elasticsearch connection timeout, exiting now...')
    sys.exit(1)

def get_cidr_from_user_input():
    """Parses one CIDR from user input and returns IPNetwork"""
    ip_or_cidr = '0'
    while not isinstance(ip_or_cidr, IPNetwork):
        try:
            ip_or_cidr = IPNetwork(input('IP/CIDR: '))
        except AddrFormatError:
            print('Not a valid IP/CIDR.')
    return ip_or_cidr

def parse_all_cidrs_from_file(file_path, assume_yes):
    """Returns set of CIDR strings from given file"""
    output = set()
    while not output:
        with open(file_path) as f:
            output = set(re.findall('(?:\d{1,3}\.){3}\d{1,3}(?:/\d\d?)?', f.read()))
            print('CIDRs Found:' + str(output))
            print('Total CIDRs in file: ' + str(len(output)))
            if not assume_yes:
                ask_continue()
    return output

def is_valid_file_name(str_input):
    """Returns if str is valid file name.
    May only contain: ascii_lowercase, ascii_uppercase, digits, dot, dash, underscore
    """
    allowed = set(string.ascii_lowercase + string.ascii_uppercase + string.digits + '.-_')
    if str_input is not '':
        return set(str_input) <= allowed
    return False

def is_valid_es_index_name(str_input):
    """Returns if str is valid Elasticsearch index name.
    May only contain: ascii_lowercase, digits, dash, underscore
    """
    allowed = set(string.ascii_lowercase + string.digits + '-_')
    if str_input is not '':
        return set(str_input) <= allowed
    return False

def dict_add_source_prefix(obj, source_str, shodan_protocol_str=''):
    """Return dict where any non-nested element (except 'ip and ip_int') is prefixed by the OSINT source name"""
    keys_not_source_prefixed = ['ip', 'asn', 'ip_int']
    # These will still have the source prefixed
    shodan_keys_not_protocol_prefixed = ['asn', 'ip', 'ipv6 port', 'hostnames', 'domains', 'location',
                                                'location.area_code', 'location.city', 'location.country_code',
                                         'location.country_code3', 'location.country_name', 'location.dma_code',
                                         'location.latitude', 'location.longitude', 'location.postal_code',
                                         'location.region_code', 'opts', 'org', 'isp', 'os', 'transport', 'protocols']
    for key in list(obj):
        # prefix all non-nested elements except ip and ip_int
        if '.' not in key and key not in keys_not_source_prefixed:
            # if other OSINT than Shodan, just prefix source
            if shodan_protocol_str is '':
                new_key = key.replace(key, (source_str + "." + key))
            # if shodan
            else:
                # just prefix source if general shodan key
                if key in shodan_keys_not_protocol_prefixed:
                    new_key = key.replace(key, (source_str + "." + key))
                # prefix source AND shodan.module (protocol) if protocol-specific key
                else:
                    new_key = key.replace(key, (source_str + "." + shodan_protocol_str + '.' + key))
            if new_key != key:
                obj[new_key] = obj[key]
                del obj[key]
    return obj

def print_json_tree(df, indent='  '):
    """Prints tree structure of given dict for test/debug purposes"""
    for key in df.keys():
        print(indent+str(key))
        if isinstance(df[key], dict):
            print_json_tree(df[key], indent + '   ')

def dict_clean_empty(d):
    """Returns dict with all empty elements removed, value 0 retained"""
    if not isinstance(d, (dict, list)):
        return d
    if isinstance(d, list):
        return [v for v in (dict_clean_empty(v) for v in d) if v]
    return {k: v for k, v in ((k, dict_clean_empty(v)) for k, v in d.items()) if v or v == 0}

def ask_input_file(path_prefix=''):
    """Returns existing file from user input"""
    input_file = Path('')
    input_file_path = ''
    while not input_file.is_file():
        input_file_path = input('Input file:' + path_prefix)
        input_file = Path(path_prefix + input_file_path)
    return input_file

def ask_input_directory():
    """Returns existing directory from user input"""
    input_directory = ''
    while not os.path.isdir(input_directory):
        input_directory = input('Input directory:')
    return input_directory

def ask_output_file(str_prefix_output_file):
    """Returns valid file path string for new file from user input"""
    str_name_output_file = ''
    while not is_valid_file_name(str_name_output_file):
        str_name_output_file = input('Output file:' + str_prefix_output_file)
    str_output_file = str_prefix_output_file + str_name_output_file
    return increment_until_new_file(str_output_file)

def increment_until_new_file(str_file):
    """Will increment given file path with number until file path does not exist yet"""
    i = 0
    str_final_file = str_file
    while os.path.exists(str_final_file):
        i += 1
        str_final_file = os.path.dirname(str_file) + '/' + os.path.splitext(os.path.basename(str(str_file)))[
            0] + str(i) + os.path.splitext(os.path.basename(str(str_file)))[1]
    return str_final_file

def get_organizations_from_csv(str_path_csv_file):
    """Returns OrderedDict containing organizations with CIDRS from given CSV file"""
    list_organizations = OrderedDict()
    f = open(str_path_csv_file, 'r')
    for line in f:
        line = line.split(',')
        if line[0] not in list_organizations:
            list_organizations[line[0]] = [line[1]]
        else:
            list_organizations[line[0]].append(line[1])
    return list_organizations

class ConcatJSONDecoder(json.JSONDecoder):
    """Returns list of dicts from given string containing multiple root JSON objects"""
    # shameless copy paste from element/decoder.py
    FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
    WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)

    def decode(self, s, _w=WHITESPACE.match):
        s_len = len(s)
        objs = []
        end = 0
        while end != s_len:
            obj, end = self.raw_decode(s, idx=_w(s, end).end())
            end = _w(s, end).end()
            objs.append(obj)
        return objs

def ask_continue():
    """Asks user if script should continue or stop immediately"""
    if get_user_boolean('Continue? y/n') is False:
        sys.exit(0)

def get_user_boolean(text):
    """Returns boolean from user input. Accepts only y (True) or n (False)"""
    while True:
        str_should_convert = input(text)
        if str_should_convert is 'y':
            return True
        elif str_should_convert is 'n':
            return False

def get_option_from_user(question, list_str_options):
    """Asks user input with given question, returns one of given options"""
    answer = ''
    while answer not in list_str_options:
        answer = input(question)
    return answer

def create_output_directory(input_directory):
    """Creates new directory and returns its path"""
    output_directory = ''
    increment = 0
    done_creating_directory = False
    while not done_creating_directory:
        try:
            if input_directory.endswith('/'):
                output_directory = input_directory + 'converted'
            else:
                output_directory = input_directory + '/converted'
            if increment is not 0:
                output_directory += str(increment)
            os.makedirs(output_directory, exist_ok=False)
            done_creating_directory = True
        except FileExistsError:
            increment += 1
    return output_directory

def get_queries_per_line_from_file(str_path_input_file):
    """Returns list of queries as string, without any blank lines"""
    with open(str_path_input_file) as f_in:
        return list(filter(None, (line.rstrip() for line in f_in)))

def check_exists_input_file(str_file_path):
    """Checks if input file exists"""
    if not Path(str_file_path).is_file():
        msg = "{0} is not an existing file".format(str_file_path)
        raise argparse.ArgumentTypeError(msg)

def get_input_choice(args):
    """Check if an input choice is given"""
    try:
        return args.subparser
    except AttributeError:
        msg = 'Missing input choice.'
        raise argparse.ArgumentTypeError(msg)

def check_outputfile(str_file_path):
    """Check if output file can be created"""
    try:
        open(str_file_path, "a")
    except FileNotFoundError:
        msg = 'Cannot create outputfile. Do all directories in outputfile path exist?'
        raise argparse.ArgumentTypeError(msg)

def get_path_converted_output_file(str_path_input_file):
    """Returns a path for converted outputfile"""
    input_file = Path(str_path_input_file)
    return increment_until_new_file('converted_outputfiles/' + os.path.splitext(os.path.basename(str(input_file)))[0]
                                    + '-converted' + os.path.splitext(str(input_file))[1])

def convert_file(str_path_input_file, source_type):
    """Converts given inputfile to outputfile"""
    from shodanfunctions import shodan_to_es_convert
    from censysfunctions import censys_to_es_convert
    from ipinfofunctions import ipinfo_to_es_convert
    str_path_output_file = get_path_converted_output_file(str_path_input_file)
    with open(str_path_output_file, 'a', encoding='utf-8') as output_file:
        input_file = Path(str_path_input_file)
        for str_banner in input_file.open(encoding='utf-8'):
            if str_banner != '\n':
                try:
                    banner = dict_clean_empty(json.loads(str_banner))
                    if source_type is 'shodan':
                        shodan_to_es_convert(banner)
                    elif source_type is 'censys':
                        censys_to_es_convert(banner)
                    elif source_type is 'ipinfo':
                        ipinfo_to_es_convert(banner)
                    output_file.write(json.dumps(banner) + '\n')
                except json.decoder.JSONDecodeError as e:
                    print(e.args)
                    print('Malformed json: ' + str_banner)
    print('Converted ' + str_path_input_file + ' to ' + str_path_output_file)

<commit_msg>base.get_organizations_from_csv will return the cidrs as IPNetworks<commit_after>from netaddr import IPNetwork, AddrFormatError
from elasticsearch import Elasticsearch
from elasticsearch import exceptions
from collections import OrderedDict
from pathlib import Path
import configparser
import argparse
import string
import json
import sys
import os
import re

def get_es_cluster_ip():
    """Returns the Elasticsearch IP from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['ELASTICSEARCH_IP']

def get_xpack_credentials():
    """Returns x-pack credentials from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['X-PACK_USERNAME'], config['elastic']['X-PACK_PASSWORD']

def xpack_enabled():
    """Returns whether x-pack is enabled from config.ini"""
    config = configparser.ConfigParser()
    config.read(os.path.dirname(os.path.realpath(__file__)) + "/config.ini")
    return config['elastic']['X-PACK_ENABLED']

def get_es_object():
    """Returns Elasticsearch object"""
    if xpack_enabled():
        credentials = get_xpack_credentials()
        return Elasticsearch([get_es_cluster_ip()], http_auth=(credentials[0], credentials[1]))
    else:
        return Elasticsearch(([{'host': get_es_cluster_ip()}]))

def es_get_all_ips(str_existing_index):
    """Returns list of list_of_ips stored in given Elasticsearch index"""
    list_ips = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"size": 0, "aggs": {"all_ip": {"terms": {"field": "ip", "size": count}}}})
    for key in res['aggregations']['all_ip']['buckets']:
        list_ips.append(key['key'])
    print('Found ' + str(len(list_ips)) + ' IPs in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return list_ips

def es_get_ips_by_query(str_existing_index):
    """Returns list of ips from query stored in given Elasticsearch index
    The query body in this function needs to be edited hardcoded.
    """
    list_ips = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"size": 0, "aggs": {"ips_by_query": {"terms": {"field": "ip", "size": count}}}, "query":
                        {"query_string": {"query":"\"cisco-IOS\" OR \" Cisco Systems\"", "analyze_wildcard": "true"}}})
    for key in res['aggregations']['ips_by_query']['buckets']:
        list_ips.append(key['key'])
    print('Found ' + str(len(list_ips)) + ' IPs by query in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return list_ips

def es_get_all(str_existing_index):
    """Returns all documents stored in given Elasticsearch index"""
    documents = []
    es = get_es_object()
    count = es.count(index=str_existing_index)['count']
    res = es.search(index=str_existing_index,
                    body={"query": { "match_all": {}}, "size": count})
    for key in res['hits']['hits']:
        documents.append(key)
    print('Found ' + str(len(documents)) + ' IPs in Elasticsearch index ' + str_existing_index)
    ask_continue()
    return documents

def exists_es_index(str_valid_index):
    """Returns if given index exists in Elasticsearch cluster"""
    connection_attempts = 0
    while connection_attempts < 3:
        try:
            es = get_es_object()
            es_indices = es.indices
            return es_indices.exists(index=str_valid_index)
        except exceptions.ConnectionTimeout:
            connection_attempts += 1
    print('Elasticsearch connection timeout, exiting now...')
    sys.exit(1)

def get_cidr_from_user_input():
    """Parses one CIDR from user input and returns IPNetwork"""
    ip_or_cidr = '0'
    while not isinstance(ip_or_cidr, IPNetwork):
        try:
            ip_or_cidr = IPNetwork(input('IP/CIDR: '))
        except AddrFormatError:
            print('Not a valid IP/CIDR.')
    return ip_or_cidr

def parse_all_cidrs_from_file(file_path, assume_yes):
    """Returns set of CIDR strings from given file"""
    output = set()
    while not output:
        with open(file_path) as f:
            output = set(re.findall('(?:\d{1,3}\.){3}\d{1,3}(?:/\d\d?)?', f.read()))
            print('CIDRs Found:' + str(output))
            print('Total CIDRs in file: ' + str(len(output)))
            if not assume_yes:
                ask_continue()
    return output

def is_valid_file_name(str_input):
    """Returns if str is valid file name.
    May only contain: ascii_lowercase, ascii_uppercase, digits, dot, dash, underscore
    """
    allowed = set(string.ascii_lowercase + string.ascii_uppercase + string.digits + '.-_')
    if str_input is not '':
        return set(str_input) <= allowed
    return False

def is_valid_es_index_name(str_input):
    """Returns if str is valid Elasticsearch index name.
    May only contain: ascii_lowercase, digits, dash, underscore
    """
    allowed = set(string.ascii_lowercase + string.digits + '-_')
    if str_input is not '':
        return set(str_input) <= allowed
    return False

def dict_add_source_prefix(obj, source_str, shodan_protocol_str=''):
    """Return dict where any non-nested element (except 'ip and ip_int') is prefixed by the OSINT source name"""
    keys_not_source_prefixed = ['ip', 'asn', 'ip_int']
    # These will still have the source prefixed
    shodan_keys_not_protocol_prefixed = ['asn', 'ip', 'ipv6 port', 'hostnames', 'domains', 'location',
                                                'location.area_code', 'location.city', 'location.country_code',
                                         'location.country_code3', 'location.country_name', 'location.dma_code',
                                         'location.latitude', 'location.longitude', 'location.postal_code',
                                         'location.region_code', 'opts', 'org', 'isp', 'os', 'transport', 'protocols']
    for key in list(obj):
        # prefix all non-nested elements except ip and ip_int
        if '.' not in key and key not in keys_not_source_prefixed:
            # if other OSINT than Shodan, just prefix source
            if shodan_protocol_str is '':
                new_key = key.replace(key, (source_str + "." + key))
            # if shodan
            else:
                # just prefix source if general shodan key
                if key in shodan_keys_not_protocol_prefixed:
                    new_key = key.replace(key, (source_str + "." + key))
                # prefix source AND shodan.module (protocol) if protocol-specific key
                else:
                    new_key = key.replace(key, (source_str + "." + shodan_protocol_str + '.' + key))
            if new_key != key:
                obj[new_key] = obj[key]
                del obj[key]
    return obj

def print_json_tree(df, indent='  '):
    """Prints tree structure of given dict for test/debug purposes"""
    for key in df.keys():
        print(indent+str(key))
        if isinstance(df[key], dict):
            print_json_tree(df[key], indent + '   ')

def dict_clean_empty(d):
    """Returns dict with all empty elements removed, value 0 retained"""
    if not isinstance(d, (dict, list)):
        return d
    if isinstance(d, list):
        return [v for v in (dict_clean_empty(v) for v in d) if v]
    return {k: v for k, v in ((k, dict_clean_empty(v)) for k, v in d.items()) if v or v == 0}

def ask_input_file(path_prefix=''):
    """Returns existing file from user input"""
    input_file = Path('')
    input_file_path = ''
    while not input_file.is_file():
        input_file_path = input('Input file:' + path_prefix)
        input_file = Path(path_prefix + input_file_path)
    return input_file

def ask_input_directory():
    """Returns existing directory from user input"""
    input_directory = ''
    while not os.path.isdir(input_directory):
        input_directory = input('Input directory:')
    return input_directory

def ask_output_file(str_prefix_output_file):
    """Returns valid file path string for new file from user input"""
    str_name_output_file = ''
    while not is_valid_file_name(str_name_output_file):
        str_name_output_file = input('Output file:' + str_prefix_output_file)
    str_output_file = str_prefix_output_file + str_name_output_file
    return increment_until_new_file(str_output_file)

def increment_until_new_file(str_file):
    """Will increment given file path with number until file path does not exist yet"""
    i = 0
    str_final_file = str_file
    while os.path.exists(str_final_file):
        i += 1
        str_final_file = os.path.dirname(str_file) + '/' + os.path.splitext(os.path.basename(str(str_file)))[
            0] + str(i) + os.path.splitext(os.path.basename(str(str_file)))[1]
    return str_final_file

def get_organizations_from_csv(str_path_csv_file):
    """Returns OrderedDict containing organizations with CIDRS from given CSV file"""
    list_organizations = OrderedDict()
    f = open(str_path_csv_file, 'r')
    for line in f:
        line = line.split(',')
        org_name = line[0]
        cidr = IPNetwork(line[1])
        if org_name not in list_organizations:
            list_organizations[org_name] = [cidr]
        else:
            list_organizations[org_name].append(cidr)
    return list_organizations

class ConcatJSONDecoder(json.JSONDecoder):
    """Returns list of dicts from given string containing multiple root JSON objects"""
    # shameless copy paste from element/decoder.py
    FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
    WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)

    def decode(self, s, _w=WHITESPACE.match):
        s_len = len(s)
        objs = []
        end = 0
        while end != s_len:
            obj, end = self.raw_decode(s, idx=_w(s, end).end())
            end = _w(s, end).end()
            objs.append(obj)
        return objs

def ask_continue():
    """Asks user if script should continue or stop immediately"""
    if get_user_boolean('Continue? y/n') is False:
        sys.exit(0)

def get_user_boolean(text):
    """Returns boolean from user input. Accepts only y (True) or n (False)"""
    while True:
        str_should_convert = input(text)
        if str_should_convert is 'y':
            return True
        elif str_should_convert is 'n':
            return False

def get_option_from_user(question, list_str_options):
    """Asks user input with given question, returns one of given options"""
    answer = ''
    while answer not in list_str_options:
        answer = input(question)
    return answer

def create_output_directory(input_directory):
    """Creates new directory and returns its path"""
    output_directory = ''
    increment = 0
    done_creating_directory = False
    while not done_creating_directory:
        try:
            if input_directory.endswith('/'):
                output_directory = input_directory + 'converted'
            else:
                output_directory = input_directory + '/converted'
            if increment is not 0:
                output_directory += str(increment)
            os.makedirs(output_directory, exist_ok=False)
            done_creating_directory = True
        except FileExistsError:
            increment += 1
    return output_directory

def get_queries_per_line_from_file(str_path_input_file):
    """Returns list of queries as string, without any blank lines"""
    with open(str_path_input_file) as f_in:
        return list(filter(None, (line.rstrip() for line in f_in)))

def check_exists_input_file(str_file_path):
    """Checks if input file exists"""
    if not Path(str_file_path).is_file():
        msg = "{0} is not an existing file".format(str_file_path)
        raise argparse.ArgumentTypeError(msg)

def get_input_choice(args):
    """Check if an input choice is given"""
    try:
        return args.subparser
    except AttributeError:
        msg = 'Missing input choice.'
        raise argparse.ArgumentTypeError(msg)

def check_outputfile(str_file_path):
    """Check if output file can be created"""
    try:
        open(str_file_path, "a")
    except FileNotFoundError:
        msg = 'Cannot create outputfile. Do all directories in outputfile path exist?'
        raise argparse.ArgumentTypeError(msg)

def get_path_converted_output_file(str_path_input_file):
    """Returns a path for converted outputfile"""
    input_file = Path(str_path_input_file)
    return increment_until_new_file('converted_outputfiles/' + os.path.splitext(os.path.basename(str(input_file)))[0]
                                    + '-converted' + os.path.splitext(str(input_file))[1])

def convert_file(str_path_input_file, source_type):
    """Converts given inputfile to outputfile"""
    from shodanfunctions import shodan_to_es_convert
    from censysfunctions import censys_to_es_convert
    from ipinfofunctions import ipinfo_to_es_convert
    str_path_output_file = get_path_converted_output_file(str_path_input_file)
    with open(str_path_output_file, 'a', encoding='utf-8') as output_file:
        input_file = Path(str_path_input_file)
        for str_banner in input_file.open(encoding='utf-8'):
            if str_banner != '\n':
                try:
                    banner = dict_clean_empty(json.loads(str_banner))
                    if source_type is 'shodan':
                        shodan_to_es_convert(banner)
                    elif source_type is 'censys':
                        censys_to_es_convert(banner)
                    elif source_type is 'ipinfo':
                        ipinfo_to_es_convert(banner)
                    output_file.write(json.dumps(banner) + '\n')
                except json.decoder.JSONDecodeError as e:
                    print(e.args)
                    print('Malformed json: ' + str_banner)
    print('Converted ' + str_path_input_file + ' to ' + str_path_output_file)

<|endoftext|>


<commit_before># BLAT tools
# based on Sri's code

import subprocess
import os

hg_idx = '~/genome/hg19.2bit'

def start_gfServer(file2idx,tileSize=7,stepSize=2,minMatch=1,maxGap=4):
    params = (tileSize,stepSize,minMatch,maxGap,file2idx)
    cmd = "gfServer -tileSize=%i -stepSize=%i -minMatch=%i -maxGap=%i start localhost 17779 %s" % params
    p = subprocess.Popen(cmd,shell=True)
    return p

def stop_gfServer(p):
    cmd = "gfServer stop localhost 17779"
    q = subprocess.Popen(cmd,shell=True)
    return q

def poll_sequence(seq,file2idx,minScore=10,minIdentity=70):
    nibdir = os.path.dirname(file2idx)
    params = (minScore,minIdentity,nibdir)
    cmd = "gfClient -minScore=%i -minIdentity=%i -nohead localhost 17779 %s /dev/stdin /dev/stdout" % params
    import sys
    print "About to initiate command"
    sys.stdout.flush()
    p = subprocess.Popen(cmd,shell=True,stdin=subprocess.PIPE,stdout=subprocess.PIPE)
    print "Initiated command.  Will now communicate with process."
    sys.stdout.flush()
    import cStringIO
    p.stdin.write(cStringIO.StringIO(">query\n%s" % seq).read())
    # p.communicate(">query\n%s" % seq)
    print "Finished communicating.  Waiting now..."
    sys.stdout.flush()

    num = 0
    for line in p.stdout:
        print "Reading output..."
        sys.stdout.flush()
        num += 1

    print "Finished with output.  Returning"
    sys.stdout.flush()

    return num
<commit_msg>Debugging<commit_after># BLAT tools
# based on Sri's code

import subprocess
import os

hg_idx = '~/genome/hg19.2bit'

def start_gfServer(file2idx,tileSize=7,stepSize=2,minMatch=1,maxGap=4):
    params = (tileSize,stepSize,minMatch,maxGap,file2idx)
    cmd = "gfServer -tileSize=%i -stepSize=%i -minMatch=%i -maxGap=%i start localhost 17779 %s" % params
    p = subprocess.Popen(cmd,shell=True)
    return p

def stop_gfServer(p):
    cmd = "gfServer stop localhost 17779"
    q = subprocess.Popen(cmd,shell=True)
    return q

def poll_sequence(seq,file2idx,minScore=10,minIdentity=70):
    nibdir = os.path.dirname(file2idx)
    params = (minScore,minIdentity,nibdir)
    cmd = "gfClient -minScore=%i -minIdentity=%i -nohead localhost 17779 %s /dev/stdin /dev/stdout" % params
    import sys
    print "About to initiate command"
    sys.stdout.flush()
    p = subprocess.Popen(cmd,shell=True,stdin=subprocess.PIPE,stdout=subprocess.PIPE)
    print "Initiated command.  Will now communicate with process."
    sys.stdout.flush()
    import cStringIO
    p.stdin.write(cStringIO.StringIO(">query\n%s" % seq).read())
    # p.communicate(">query\n%s" % seq)
    print "Finished communicating.  Waiting now..."
    sys.stdout.flush()

    print p.stdout.read()

    num = 0
    for line in p.stdout:
        print "Reading output..."
        sys.stdout.flush()
        num += 1

    print "Finished with output.  Returning"
    sys.stdout.flush()

    return num
<|endoftext|>


<commit_before>"""
Evolution of ByteBeat Algorithms
Charles Martin
November 2016
"""
from __future__ import print_function
from deap import base, creator, gp, tools, algorithms
import operator
import numpy as np

def beat(t):
    return  t*(t+(t>>9|t>>13))%40&120

def playback_expr(e):
    t = 1
    while True:
        print(chr(int(
            e(t)
        ) % 256 ), end="")
        t += 1

def playback_expr_count(e):
    for t in range(50000):
        print(chr(int(
            e(t+1)
        ) % 256 ), end="")
        t += 1

def playback_char(e,t):
    return (int(e(t+1)) % 256)

def gen_beat_output(e):
    return [playback_char(e,t) for t in range(1000)]

"""
Setup the Evolutionary Programming system
"""

pset = gp.PrimitiveSet("MAIN", 1)
pset.addPrimitive(operator.add, 2)
pset.addPrimitive(operator.mod, 2)
pset.addPrimitive(operator.mul, 2)
pset.addPrimitive(operator.rshift,2)
pset.addPrimitive(operator.lshift,2)
pset.addPrimitive(operator.or_,2)
pset.addPrimitive(operator.and_,2)
pset.addPrimitive(operator.xor,2)
#pset.addPrimitive(operator.sub,2)
pset.addPrimitive(operator.floordiv,2)
#pset.addTerminal(1)
#pset.addTerminal(2)
#pset.addTerminal(3)
pset.addTerminal(5)
pset.addTerminal(7)
pset.renameArguments(ARG0='t')

def make_random_beat():
    expr = gp.genFull(pset, min_=3,max_=10)
    tree = gp.PrimitiveTree(expr)
    #print(str(tree))
    return gp.compile(tree,pset)

def make_test_tree():
    expr = gp.genFull(pset, min_=3,max_=10)
    tree = gp.PrimitiveTree(expr)
    return tree

def evalBeat(individual):
    # compile the individual
    routine = gp.compile(individual, pset)
    # generate some test output
    try:
        test_output = gen_beat_output(routine)
    except:
        return 0
    ## do some stats on the beat
    sd = np.std(test_output)
    # return the score
    return sd,

def make_test_output():
    out = []
    try:
        f = make_random_beat()
        out = gen_beat_output(f)
    except:
        print("failed")
    return out

"""
blippy, epic: (t*((15&t>>11)%12)&55-(t>>5|t>>12)|t*(t>>10)*32)-1
atmospheric, hopeful: t*3&(t>>10)|t*12&(t>>10)|t*10&((t>>8)*55)&128
expansive rumbles: t*4&(t>>10)|t*4&(t*6>>8)&t|64
electric, repetitive: t*(t+(t>>9|t>>13))%40&120

"""
def playback_random_beat():
    a = ""
    while True:
        try:
            f = make_random_beat()
            playback_expr_count(f)
        except ValueError:
            a = "Value Error"
        except ZeroDivisionError:
            a = "Zero Div Error"
        except:
            a = "other error"
        print(a)

"""
Setup the GP evolution
"""

creator.create("FitnessMax", base.Fitness, weights = (1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMax)

toolbox = base.Toolbox()
toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=3, max_=10)
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("compile", gp.compile, pset=pset)

toolbox.register("evaluate", evalBeat)
toolbox.register("select", tools.selTournament, tournsize=3)
toolbox.register("mate", gp.cxOnePoint)
toolbox.register("expr_mut", gp.genFull, min_=0, max_=6)
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)

toolbox.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))
toolbox.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))

def main():
    random.seed(318)

    # Stats for population:
    stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
    stats_size = tools.Statistics(len)
    mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
    mstats.register("avg", np.mean)
    mstats.register("std", np.std)
    mstats.register("min", np.min)
    mstats.register("max", np.max)

    pop = toolbox.population(n=100)
    hof = tools.HallOfFame(10)
    pop, log = algorithms.eaSimple(pop, toolbox, 0.5, 0.1, 20, stats=mstats,
                                   halloffame=hof, verbose=True)
    # print log
    return pop, log, hof

if __name__ == "__main__":
    main()

<commit_msg>working<commit_after>"""
Evolution of ByteBeat Algorithms
Charles Martin
November 2016
"""
from __future__ import print_function
from deap import base, creator, gp, tools, algorithms
import operator
import numpy as np
import random

def beat(t):
    return  t*(t+(t>>9|t>>13))%40&120

def playback_expr(e):
    t = 1
    while True:
        print(chr(int(
            e(t)
        ) % 256 ), end="")
        t += 1

def playback_expr_count(e):
    for t in range(50000):
        print(chr(int(
            e(t+1)
        ) % 256 ), end="")
        t += 1

def playback_char(e,t):
    return (int(e(t+1)) % 256)

def gen_beat_output(e):
    return [playback_char(e,t) for t in range(100)]

"""
Setup the Evolutionary Programming system
"""

pset = gp.PrimitiveSet("MAIN", 1)
pset.addPrimitive(operator.add, 2)
pset.addPrimitive(operator.mod, 2)
pset.addPrimitive(operator.mul, 2)
pset.addPrimitive(operator.rshift,2)
pset.addPrimitive(operator.lshift,2)
pset.addPrimitive(operator.or_,2)
pset.addPrimitive(operator.and_,2)
pset.addPrimitive(operator.xor,2)
#pset.addPrimitive(operator.sub,2)
pset.addPrimitive(operator.floordiv,2)
#pset.addTerminal(1)
#pset.addTerminal(2)
#pset.addTerminal(3)
pset.addTerminal(5)
pset.addTerminal(7)
pset.renameArguments(ARG0='t')

def make_random_beat():
    expr = gp.genFull(pset, min_=3,max_=10)
    tree = gp.PrimitiveTree(expr)
    #print(str(tree))
    return gp.compile(tree,pset)

def make_test_tree():
    expr = gp.genFull(pset, min_=3,max_=10)
    tree = gp.PrimitiveTree(expr)
    return tree

def evalBeat(individual):
    # compile the individual
    routine = gp.compile(individual, pset)
    # generate some test output
    try:
        test_output = gen_beat_output(routine)
    except:
        return 0.0,
    ## do some stats on the beat
    sd = np.std(np.array(test_output))
    # return the score
    return float(sd),

def make_test_output():
    out = []
    try:
        f = make_random_beat()
        out = gen_beat_output(f)
    except:
        print("failed")
    return out

"""
blippy, epic: (t*((15&t>>11)%12)&55-(t>>5|t>>12)|t*(t>>10)*32)-1
atmospheric, hopeful: t*3&(t>>10)|t*12&(t>>10)|t*10&((t>>8)*55)&128
expansive rumbles: t*4&(t>>10)|t*4&(t*6>>8)&t|64
electric, repetitive: t*(t+(t>>9|t>>13))%40&120

"""
def playback_random_beat():
    a = ""
    while True:
        try:
            f = make_random_beat()
            playback_expr_count(f)
        except ValueError:
            a = "Value Error"
        except ZeroDivisionError:
            a = "Zero Div Error"
        except:
            a = "other error"
        print(a)

"""
Setup the GP evolution
"""

creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMax)

toolbox = base.Toolbox()
# Attribute generator
toolbox.register("expr_init", gp.genFull, pset=pset, min_=1, max_=2)

# Structure initializers
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr_init)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)

toolbox.register("evaluate", evalBeat)
toolbox.register("select", tools.selTournament, tournsize=7)
toolbox.register("mate", gp.cxOnePoint)
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)

def main():
    random.seed(318)

    pop = toolbox.population(n=20)
    hof = tools.HallOfFame(1)
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("avg", np.mean)
    stats.register("std", np.std)
    stats.register("min", np.min)
    stats.register("max", np.max)

    print("Starting EA Simple")
    algorithms.eaSimple(pop, toolbox, 0.5, 0.2, 10, stats, halloffame=hof)

    return pop, hof, stats

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>"""CLC.PY: A SIMPLE PYTHON CALCULATOR."""
import sys

def add_all(nums):
    return sum(nums)

def multiply_all(nums):
    return reduce(lambda a, b: a * b, nums)

if __name__ == '__main__':
    command = sys.argv[1]
    nums = map(float, sys.argv[2:])
    if command == 'add':
        print(add_all(nums))
    elif command == 'multiply':
        print(multiply_all(nums))
<commit_msg>Add suport for min<commit_after>"""CLC.PY: A SIMPLE PYTHON CALCULATOR."""
import sys

def add_all(nums):
    return sum(nums)

def multiply_all(nums):
    return reduce(lambda a, b: a * b, nums)

if __name__ == '__main__':
    command = sys.argv[1]
    nums = map(float, sys.argv[2:])
    if command == 'add':
        print(add_all(nums))
    elif command == 'multiply':
        print(multiply_all(nums))
    elif command == 'min':
        print(min(nums))
<|endoftext|>


<commit_before>import sys
#def
if __name__ == '__main__':

	print(sum(map(float, sys.argv[1:])))<commit_msg>Added another comment line<commit_after>import sys
#def

#deedede

if __name__ == '__main__':

	print(sum(map(float, sys.argv[1:])))<|endoftext|>


<commit_before>from functools import wraps
from flask import Flask, g, request, render_template, make_response, jsonify, abort

from lib import PING_PERIOD, ARCHIVE_PERIOD, get_time
from lib.messages import send_message, get_user_list, parse_messages
from lib.requests import connect_redis, create_chat_session, set_cookie, disconnect_redis
from lib.sessions import get_counter

app = Flask(__name__)

# Pre and post request stuff
app.before_request(connect_redis)
app.before_request(create_chat_session)
app.after_request(set_cookie)
app.after_request(disconnect_redis)

# Helper functions

def get_wanted_channels(channel_main, channel_mod, channel_self):
    wanted_channels = set()
    wanted_channels.add(channel_main)
    if g.user.group=='mod':
        # Moderator messages.
        wanted_channels.add(channel_mod)
    if g.user.group=='silent':
        # Channel for self messages if silent.
        wanted_channels.add(channel_self)
    return wanted_channels

# Decorators

def mark_alive(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        chat = request.form['chat']
        state_key = 'chat.%s.sessions' % chat
        session_state = g.redis.hget(state_key, g.user.session)
        if session_state is None:
            # This session has never been in this chat before; we need to add them to the counter.
            g.redis.rpush('chat.%s.counter' % chat, g.user.session)
            g.redis.sadd('session.%s.chats' % g.user.session, chat)
        if session_state not in ['online', 'away']:
            # Remove from the delete queue and add to the archive queue.
            g.redis.zrem('delete-queue', chat)
            if g.redis.zscore('archive-queue', chat) is None:
                g.redis.zadd('archive-queue', chat, get_time(ARCHIVE_PERIOD))
            # Set user state.
            g.redis.hset(state_key, g.user.session, 'online')
            if g.user.group=='silent':
                join_message = None
                g.fake_join_message = True
            else:
                join_message = '%s [%s] joined chat.' % (g.user.name, g.user.acronym)
            send_message(g.redis, chat, -1, 'user_change', join_message)
            g.redis.sadd('sessions-chatting', g.user.session)
            # Add character to chat character list.
            g.redis.sadd('chat.'+chat+'.characters', g.user.character)
        g.redis.zadd('chats-alive', chat+'/'+g.user.session, get_time(PING_PERIOD*2))
        return f(*args, **kwargs)
    return decorated_function

# Views

@app.route('/post', methods=['POST'])
@mark_alive
def postMessage():
    chat = request.form['chat']
    if 'line' in request.form:
        # Remove linebreaks and truncate to 1500 characters.
        line = request.form['line'].replace('\n', ' ')[:1500]
        counter = get_counter(chat, g.user.session)
        if g.user.group=='silent':
            send_message(g.redis, chat, counter, 'private', line, g.user.color, g.user.acronym, g.user.session)
        else:
            send_message(g.redis, chat, counter, 'message', line, g.user.color, g.user.acronym)
    if 'state' in request.form and request.form['state'] in ['online', 'away']:
        current_state = g.redis.hget('chat.%s.sessions' % chat, g.user.session)
        if request.form['state']!=current_state:
            g.redis.hset('chat.%s.sessions' % chat, g.user.session, request.form['state'])
            if request.form['state']=='away':
                send_message(g.redis, chat, -1, 'user_change')
            else:
                send_message(g.redis, chat, -1, 'user_change')
    if 'set_group' in request.form and 'counter' in request.form:
        if g.user.group=='mod':
            set_group = request.form['set_group']
            set_session_id = g.redis.lindex('chat.%s.counter' % chat, request.form['counter']) or abort(400)
            set_session_key = 'session.%s.chat.%s' % (set_session_id, chat)
            set_session = g.redis.hgetall(set_session_key)
            if set_session['group']!=set_group and set_group in ['user', 'mod', 'silent']:
                g.redis.hset(set_session_key, 'group', set_group)
                set_message = None
                # Convert the name and acronym to unicode.
                set_session['name'] = unicode(set_session['name'], encoding='utf8')
                set_session['acronym'] = unicode(set_session['acronym'], encoding='utf8')
                if set_session['group']!='mod' and set_group=='mod':
                    set_message = '%s [%s] gave moderator status to %s [%s].' % (g.user.name, g.user.acronym, set_session['name'], set_session['acronym'])
                elif set_session['group']=='mod' and set_group!='mod':
                    set_message = '%s [%s] removed moderator status from %s [%s].' % (g.user.name, g.user.acronym, set_session['name'], set_session['acronym'])
                # Refresh the user's subscriptions.
                g.redis.publish('channel.'+chat+'.refresh', set_session_id+'#'+set_group)
                send_message(g.redis, chat, -1, 'user_change', set_message)
        else:
            abort(403)
    return 'ok'

@app.route('/ping', methods=['POST'])
@mark_alive
def pingServer():
    return 'ok'

@app.route('/messages', methods=['POST'])
@mark_alive
def getMessages():

    chat = request.form['chat']
    after = int(request.form['after'])

    message_dict = None

    if hasattr(g, 'fake_join_message'):
        message_dict = { 'messages': [ {
            'id': after,
            'timestamp': get_time(),
            'counter': -1,
            'color': '000000',
            'line': '%s [%s] joined chat.' % (g.user.name, g.user.acronym)
        } ] }
    else:
        # Check for stored messages.
        messages = g.redis.lrange('chat.'+chat, after+1, -1)
        if messages:
            message_dict = {
                'messages': parse_messages(messages, after+1)
            }

    if message_dict:
        message_dict['online'] = get_user_list(g.redis, chat, 'mod' if g.user.group=='mod' else 'user')
        if 'fetchCounter' in request.form:
            message_dict['counter'] = get_counter(chat, g.user.session)
        return jsonify(message_dict)

    # Otherwise, listen for a message.

    # Channel names.
    channel_main = 'channel.'+chat
    channel_mod = channel_main+'.mod'
    channel_self = channel_main+'.'+g.user.session
    channel_refresh = channel_main+'.refresh'

    # We subscribe to all four channels then ignore what we don't want because
    # changing subscriptions doesn't happen quickly enough and we end up missing
    # messages.
    g.redis.subscribe(channel_main)
    g.redis.subscribe(channel_mod)
    g.redis.subscribe(channel_self)
    g.redis.subscribe(channel_refresh)

    # This gives us a list of all the channels we want to listen to.
    wanted_channels = get_wanted_channels(channel_main, channel_mod, channel_self)

    for msg in g.redis.listen():
        if msg['type']=='message':
            if msg['channel']==channel_refresh:
                refresh_user, refresh_group = msg['data'].split('#', 1)
                if refresh_user==g.user.session:
                    # Our group has changed. Alter wanted channels accordingly.
                    g.user.group = refresh_group
                    wanted_channels = get_wanted_channels(channel_main, channel_mod, channel_self)
            elif msg['channel'] in wanted_channels:
                # The pubsub channel sends us a JSON string, so we return that instead of using jsonify.
                resp = make_response(msg['data'])
                resp.headers['Content-type'] = 'application/json'
                return resp

@app.route('/quit', methods=['POST'])
def quitChatting():
    # Check if they're actually a member of the chat first?
    chatkey = 'chat.%s.sessions' % request.form['chat']
    if g.redis.hexists(chatkey, g.user.session):
        g.redis.zrem('chats-alive', request.form['chat']+'/'+g.user.session)
        g.redis.hset(chatkey, g.user.session, 'offline')
        g.redis.srem('sessions-chatting', g.user.session)
        disconnect_message = '%s [%s] disconnected.' % (g.user.name, g.user.acronym) if g.user.group!='silent' else None
        send_message(g.redis, request.form['chat'], -1, 'user_change', disconnect_message)
        return 'ok'

@app.route('/save', methods=['POST'])
def save():
    try:
        g.user.save_character(request.form)
    except ValueError as e:
        abort(400)
    return 'ok'

if __name__ == "__main__":
    app.run(port=9000, debug=True)

<commit_msg>Use redis.pubsub().<commit_after>from functools import wraps
from flask import Flask, g, request, render_template, make_response, jsonify, abort

from lib import PING_PERIOD, ARCHIVE_PERIOD, get_time
from lib.messages import send_message, get_user_list, parse_messages
from lib.requests import connect_redis, create_chat_session, set_cookie, disconnect_redis
from lib.sessions import get_counter

app = Flask(__name__)

# Pre and post request stuff
app.before_request(connect_redis)
app.before_request(create_chat_session)
app.after_request(set_cookie)
app.after_request(disconnect_redis)

# Helper functions

def get_wanted_channels(channel_main, channel_mod, channel_self):
    wanted_channels = set()
    wanted_channels.add(channel_main)
    if g.user.group=='mod':
        # Moderator messages.
        wanted_channels.add(channel_mod)
    if g.user.group=='silent':
        # Channel for self messages if silent.
        wanted_channels.add(channel_self)
    return wanted_channels

# Decorators

def mark_alive(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        chat = request.form['chat']
        state_key = 'chat.%s.sessions' % chat
        session_state = g.redis.hget(state_key, g.user.session)
        if session_state is None:
            # This session has never been in this chat before; we need to add them to the counter.
            g.redis.rpush('chat.%s.counter' % chat, g.user.session)
            g.redis.sadd('session.%s.chats' % g.user.session, chat)
        if session_state not in ['online', 'away']:
            # Remove from the delete queue and add to the archive queue.
            g.redis.zrem('delete-queue', chat)
            if g.redis.zscore('archive-queue', chat) is None:
                g.redis.zadd('archive-queue', chat, get_time(ARCHIVE_PERIOD))
            # Set user state.
            g.redis.hset(state_key, g.user.session, 'online')
            if g.user.group=='silent':
                join_message = None
                g.fake_join_message = True
            else:
                join_message = '%s [%s] joined chat.' % (g.user.name, g.user.acronym)
            send_message(g.redis, chat, -1, 'user_change', join_message)
            g.redis.sadd('sessions-chatting', g.user.session)
            # Add character to chat character list.
            g.redis.sadd('chat.'+chat+'.characters', g.user.character)
        g.redis.zadd('chats-alive', chat+'/'+g.user.session, get_time(PING_PERIOD*2))
        return f(*args, **kwargs)
    return decorated_function

# Views

@app.route('/post', methods=['POST'])
@mark_alive
def postMessage():
    chat = request.form['chat']
    if 'line' in request.form:
        # Remove linebreaks and truncate to 1500 characters.
        line = request.form['line'].replace('\n', ' ')[:1500]
        counter = get_counter(chat, g.user.session)
        if g.user.group=='silent':
            send_message(g.redis, chat, counter, 'private', line, g.user.color, g.user.acronym, g.user.session)
        else:
            send_message(g.redis, chat, counter, 'message', line, g.user.color, g.user.acronym)
    if 'state' in request.form and request.form['state'] in ['online', 'away']:
        current_state = g.redis.hget('chat.%s.sessions' % chat, g.user.session)
        if request.form['state']!=current_state:
            g.redis.hset('chat.%s.sessions' % chat, g.user.session, request.form['state'])
            if request.form['state']=='away':
                send_message(g.redis, chat, -1, 'user_change')
            else:
                send_message(g.redis, chat, -1, 'user_change')
    if 'set_group' in request.form and 'counter' in request.form:
        if g.user.group=='mod':
            set_group = request.form['set_group']
            set_session_id = g.redis.lindex('chat.%s.counter' % chat, request.form['counter']) or abort(400)
            set_session_key = 'session.%s.chat.%s' % (set_session_id, chat)
            set_session = g.redis.hgetall(set_session_key)
            if set_session['group']!=set_group and set_group in ['user', 'mod', 'silent']:
                g.redis.hset(set_session_key, 'group', set_group)
                set_message = None
                # Convert the name and acronym to unicode.
                set_session['name'] = unicode(set_session['name'], encoding='utf8')
                set_session['acronym'] = unicode(set_session['acronym'], encoding='utf8')
                if set_session['group']!='mod' and set_group=='mod':
                    set_message = '%s [%s] gave moderator status to %s [%s].' % (g.user.name, g.user.acronym, set_session['name'], set_session['acronym'])
                elif set_session['group']=='mod' and set_group!='mod':
                    set_message = '%s [%s] removed moderator status from %s [%s].' % (g.user.name, g.user.acronym, set_session['name'], set_session['acronym'])
                # Refresh the user's subscriptions.
                g.redis.publish('channel.'+chat+'.refresh', set_session_id+'#'+set_group)
                send_message(g.redis, chat, -1, 'user_change', set_message)
        else:
            abort(403)
    return 'ok'

@app.route('/ping', methods=['POST'])
@mark_alive
def pingServer():
    return 'ok'

@app.route('/messages', methods=['POST'])
@mark_alive
def getMessages():

    chat = request.form['chat']
    after = int(request.form['after'])

    message_dict = None

    if hasattr(g, 'fake_join_message'):
        message_dict = { 'messages': [ {
            'id': after,
            'timestamp': get_time(),
            'counter': -1,
            'color': '000000',
            'line': '%s [%s] joined chat.' % (g.user.name, g.user.acronym)
        } ] }
    else:
        # Check for stored messages.
        messages = g.redis.lrange('chat.'+chat, after+1, -1)
        if messages:
            message_dict = {
                'messages': parse_messages(messages, after+1)
            }

    if message_dict:
        message_dict['online'] = get_user_list(g.redis, chat, 'mod' if g.user.group=='mod' else 'user')
        if 'fetchCounter' in request.form:
            message_dict['counter'] = get_counter(chat, g.user.session)
        return jsonify(message_dict)

    # Otherwise, listen for a message.

    # Channel names.
    channel_main = 'channel.'+chat
    channel_mod = channel_main+'.mod'
    channel_self = channel_main+'.'+g.user.session
    channel_refresh = channel_main+'.refresh'

    pubsub = g.redis.pubsub()

    # We subscribe to all four channels then ignore what we don't want because
    # changing subscriptions doesn't happen quickly enough and we end up missing
    # messages.
    pubsub.subscribe(channel_main)
    pubsub.subscribe(channel_mod)
    pubsub.subscribe(channel_self)
    pubsub.subscribe(channel_refresh)

    # This gives us a list of all the channels we want to listen to.
    wanted_channels = get_wanted_channels(channel_main, channel_mod, channel_self)

    for msg in pubsub.listen():
        if msg['type']=='message':
            if msg['channel']==channel_refresh:
                refresh_user, refresh_group = msg['data'].split('#', 1)
                if refresh_user==g.user.session:
                    # Our group has changed. Alter wanted channels accordingly.
                    g.user.group = refresh_group
                    wanted_channels = get_wanted_channels(channel_main, channel_mod, channel_self)
            elif msg['channel'] in wanted_channels:
                # The pubsub channel sends us a JSON string, so we return that instead of using jsonify.
                resp = make_response(msg['data'])
                resp.headers['Content-type'] = 'application/json'
                return resp

@app.route('/quit', methods=['POST'])
def quitChatting():
    # Check if they're actually a member of the chat first?
    chatkey = 'chat.%s.sessions' % request.form['chat']
    if g.redis.hexists(chatkey, g.user.session):
        g.redis.zrem('chats-alive', request.form['chat']+'/'+g.user.session)
        g.redis.hset(chatkey, g.user.session, 'offline')
        g.redis.srem('sessions-chatting', g.user.session)
        disconnect_message = '%s [%s] disconnected.' % (g.user.name, g.user.acronym) if g.user.group!='silent' else None
        send_message(g.redis, request.form['chat'], -1, 'user_change', disconnect_message)
        return 'ok'

@app.route('/save', methods=['POST'])
def save():
    try:
        g.user.save_character(request.form)
    except ValueError as e:
        abort(400)
    return 'ok'

if __name__ == "__main__":
    app.run(port=9000, debug=True)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Nottingham Hackspace Rules documentation build configuration file, created by
# sphinx-quickstart on Mon Nov  9 08:25:35 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Nottingham Hackspace Rules'
copyright = u'2015, Nottingham Hackspace'
author = u'Nottingham Hackspace'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1.0'
# The full version, including alpha/beta/rc tags.
release = '0.1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'NottinghamHackspaceRulesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',

'classoptions': ',oneside',
'babel': '\\usepackage[english]{babel}'
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'NottinghamHackspaceRules.tex', u'Nottingham Hackspace Rules Documentation',
   u'Nottingham Hackspace', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'nottinghamhackspacerules', u'Nottingham Hackspace Rules Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'NottinghamHackspaceRules', u'Nottingham Hackspace Rules Documentation',
   author, 'NottinghamHackspaceRules', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<commit_msg>Bump copyright year<commit_after># -*- coding: utf-8 -*-
#
# Nottingham Hackspace Rules documentation build configuration file, created by
# sphinx-quickstart on Mon Nov  9 08:25:35 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Nottingham Hackspace Rules'
copyright = u'2021, Nottingham Hackspace'
author = u'Nottingham Hackspace'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1.0'
# The full version, including alpha/beta/rc tags.
release = '0.1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'NottinghamHackspaceRulesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',

'classoptions': ',oneside',
'babel': '\\usepackage[english]{babel}'
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'NottinghamHackspaceRules.tex', u'Nottingham Hackspace Rules Documentation',
   u'Nottingham Hackspace', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'nottinghamhackspacerules', u'Nottingham Hackspace Rules Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'NottinghamHackspaceRules', u'Nottingham Hackspace Rules Documentation',
   author, 'NottinghamHackspaceRules', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# CMPI Standard documentation build configuration file, created by
# sphinx-quickstart on Mon Aug  3 15:11:55 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex
import subprocess, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.pngmath', # added for Breathe
    'sphinx.ext.todo',    # added for Breathe
    'breathe'             # added for Breathe
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CMPI Standard'
copyright = u'2006-2015, The Open Group'
author = u'The Open Group'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '2.1'
# The full version, including alpha/beta/rc tags.
release = '2.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['doxygen', '_build', 'sphinx-ext']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CMPIStandarddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'CMPIStandard.tex', u'CMPI Standard Documentation',
   u'The Open Group', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cmpistandard', u'CMPI Standard Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'CMPIStandard', u'CMPI Standard Documentation',
   author, 'CMPIStandard', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Breathe extension ----------------------------------------------------

# Download the Breathe 4.0.0 tarball from PyPi and expand it into ./sphinx-ext/,
# so that the 'breathe' module is found in the following path:
sys.path.append("sphinx-ext/breathe-4.0.0")

breathe_projects = {
    "cmpi-headers": "doxygen/xml/"
}
breathe_default_project = "cmpi-headers"

# -- Running Doxygen ------------------------------------------------------

print("Debug: In conf.py: CWD=%s" % os.getcwd())

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'
if read_the_docs_build:
    # We run on readthedocs.org.
    subprocess.call('cd ../doxygen; doxygen', shell=True)
else:
    # We run locally.
    subprocess.call('doxygen', shell=True)
<commit_msg>Breathe fix for involing Doxygen.<commit_after># -*- coding: utf-8 -*-
#
# CMPI Standard documentation build configuration file, created by
# sphinx-quickstart on Mon Aug  3 15:11:55 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex
import subprocess, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.pngmath', # added for Breathe
    'sphinx.ext.todo',    # added for Breathe
    'breathe'             # added for Breathe
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CMPI Standard'
copyright = u'2006-2015, The Open Group'
author = u'The Open Group'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '2.1'
# The full version, including alpha/beta/rc tags.
release = '2.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['doxygen', '_build', 'sphinx-ext']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CMPIStandarddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'CMPIStandard.tex', u'CMPI Standard Documentation',
   u'The Open Group', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cmpistandard', u'CMPI Standard Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'CMPIStandard', u'CMPI Standard Documentation',
   author, 'CMPIStandard', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Breathe extension ----------------------------------------------------

# Download the Breathe 4.0.0 tarball from PyPi and expand it into ./sphinx-ext/,
# so that the 'breathe' module is found in the following path:
sys.path.append("sphinx-ext/breathe-4.0.0")

breathe_projects = {
    "cmpi-headers": "doxygen/xml/"
}
breathe_default_project = "cmpi-headers"

# -- Running Doxygen ------------------------------------------------------

print("Debug: In conf.py: CWD=%s" % os.getcwd())

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'
if read_the_docs_build:
    # We run on readthedocs.org.
    print("Debug: ls -l ../doxygen:" % os.getcwd())
    subprocess.call('ls -l ../doxygen', shell=True)
    print("Debug: Calling Doxygen in ReadTheDocs environment")
    subprocess.call('cd ../doxygen; doxygen', shell=True)
else:
    # We run locally.
    print("Debug: Calling Doxygen in local environment")
    subprocess.call('doxygen', shell=True)
print("Debug: Doxygen done")
<|endoftext|>