<commit_before>''' -- imports from python libraries -- '''
import os
import time
import datetime
import csv
from collections import OrderedDict
# import json

''' imports from installed packages '''
from django.utils.text import slugify
from django.core.management.base import BaseCommand, CommandError
# from django.contrib.auth.models import User

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Group, GSystemType, Counter, node_collection, Buddy, Author
from gnowsys_ndf.settings import GSTUDIO_LOGS_DIR_PATH, GSTUDIO_DATA_ROOT
from gnowsys_ndf.ndf.views.gcourse import course_analytics
# from gnowsys_ndf.ndf.views.methods import get_group_name_id
# from gnowsys_ndf.ndf.views.analytics_methods import *

try:
    from gnowsys_ndf.server_settings import GSTUDIO_INSTITUTE_ID, GSTUDIO_INSTITUTE_ID_SECONDARY, GSTUDIO_INSTITUTE_NAME
except Exception, e:
    from gnowsys_ndf.settings import GSTUDIO_INSTITUTE_ID, GSTUDIO_INSTITUTE_ID_SECONDARY, GSTUDIO_INSTITUTE_NAME

if not os.path.exists(GSTUDIO_LOGS_DIR_PATH):
    os.makedirs(GSTUDIO_LOGS_DIR_PATH)

log_file_name = 'export_users_analytics.log'
log_file_path = os.path.join(GSTUDIO_LOGS_DIR_PATH, log_file_name)
log_file = open(log_file_path, 'a+')
script_start_str = "\n\n######### Script ran on : " + time.strftime("%c") + " #########\n----------------\n"
log_file.write(str(script_start_str))

column_keys_list = ["server_id", "school_name", "school_code", "unit_name", "username", "user_id", "enrollment_status", "total_lessons", "lessons_completed", "percentage_lessons_completed", "total_activities", "activities_completed", "percentage_activities_completed", "total_quizitems", "visited_quizitems", "attempted_quizitems", "unattempted_quizitems", "correct_attempted_quizitems", "notapplicable_quizitems", "incorrect_attempted_quizitems", "user_files", "total_files_viewed_by_user", "other_viewing_my_files", "unique_users_commented_on_user_files", "total_rating_rcvd_on_files", "commented_on_others_files", "cmts_on_user_files", "total_cmnts_by_user", "user_notes", "others_reading_my_notes", "cmts_on_user_notes", "cmnts_rcvd_by_user", "total_notes_read_by_user", "commented_on_others_notes", "total_rating_rcvd_on_notes", "correct_attempted_assessments", "unattempted_assessments", "visited_assessments", "notapplicable_assessments", "incorrect_attempted_assessments", "attempted_assessments", "total_assessment_items", "buddies"]

column_keys_dict = OrderedDict()
map(lambda x: column_keys_dict.update({x: "NA"}), column_keys_list)

class Command(BaseCommand):

    def handle(self, *args, **options):
        gst_base_unit_name, gst_base_unit_id = GSystemType.get_gst_name_id('announced_unit')
        gst_course_ev_gr = node_collection.one({'_type': 'GSystemType', 'name': 'CourseEventGroup'})
        all_course_groups = node_collection.find({'_type': 'Group', 'member_of': {'$in': [gst_course_ev_gr._id, gst_base_unit_id]} })

        for each_group_obj in all_course_groups:
            assessment_and_quiz_data = True
            print "\n\n Exporting CSV for :", each_group_obj.name, "(Altnames:", each_group_obj.altnames, ")\n"
            export_group_analytics(each_group_obj, assessment_and_quiz_data)

def export_group_analytics(group_obj, assessment_and_quiz_data):

    # print "Enter 'group name(case sensitive)' OR 'id': "
    # group_name_or_id = raw_input()
    # group_obj = Group.get_group_name_id(group_name_or_id, get_obj=True)

    # if not group_obj:
    #     raise ValueError('\nSorry. Request could not be completed. \
    #         \nGroup/Course, matching argument entered "' + group_name_or_id + '" does not exists!')

    # group_users = group_obj.author_set
    # Previously, above was implementation strategy. Now we are using counter users instead of group's author_set,
    # because we do wanted to gather all data irrespective of student enrollment.
    group_users = Counter.get_group_counters(group_obj._id).distinct('user_id')
    # group_users is a list of integer user ids
    # print group_users

    # CSV file name-convention: schoolcode-course-name-datetimestamp.csv
    try:
        group_name = slugify(group_obj['name'])
    except Exception, e:
        print e
        group_name = str(group_obj['_id'])

    # dt: date time
    # e.g: '21-November-2016-19h-08m-10s'
    # dt = "{:%d-%B-%Y-%Hh-%Mm-%Ss}".format(datetime.datetime.now())
    dt = "{:%Y%m%d-%Hh%Mm}".format(datetime.datetime.now())

    file_name = GSTUDIO_INSTITUTE_ID_SECONDARY + '-' + GSTUDIO_INSTITUTE_ID + '-' + group_name + '-' + dt + '.csv'

    GSTUDIO_EXPORTED_CSVS_DIRNAME = 'gstudio-exported-users-analytics-csvs'
    GSTUDIO_EXPORTED_CSVS_DIR_PATH = os.path.join('/data/', GSTUDIO_EXPORTED_CSVS_DIRNAME)

    if not os.path.exists(GSTUDIO_EXPORTED_CSVS_DIR_PATH):
        os.makedirs(GSTUDIO_EXPORTED_CSVS_DIR_PATH)

    file_name_path = os.path.join(GSTUDIO_EXPORTED_CSVS_DIR_PATH, file_name)

    header_written = False
    for index, each_user in enumerate(group_users):
        try:
            analytics_data = course_analytics(None, group_obj._id, each_user, get_result_dict=True, assessment_and_quiz_data=assessment_and_quiz_data)
            if not analytics_data:
                continue

            print index, "] Group User ID: ", each_user

            each_row_dict = column_keys_dict.copy()
            each_row_dict['server_id'] = GSTUDIO_INSTITUTE_ID
            each_row_dict['school_code'] = GSTUDIO_INSTITUTE_ID_SECONDARY
            each_row_dict['school_name'] = GSTUDIO_INSTITUTE_NAME
            each_row_dict['unit_name'] = group_name

            temp_lessons_stat_str = analytics_data['level1_progress_stmt']
            temp_lessons_stat_str = analytics_data['level1_progress_stmt']
            each_row_dict['lessons_completed'] = int(temp_lessons_stat_str.split(' ')[0])
            each_row_dict['total_lessons'] = int(temp_lessons_stat_str.split(' ')[3])

            temp_activities_stat_str = analytics_data['level2_progress_stmt']
            each_row_dict['activities_completed'] = int(temp_activities_stat_str.split(' ')[0])
            each_row_dict['total_activities'] = int(temp_activities_stat_str.split(' ')[3])

            each_row_dict['percentage_lessons_completed'] = analytics_data['level1_progress_meter']
            each_row_dict['percentage_activities_completed'] = analytics_data['level2_progress_meter']

            each_row_dict['username'] = analytics_data['username']
            each_row_dict['user_id'] = each_user
            each_row_dict['enrollment_status'] = "Yes" if (each_user in group_obj.author_set) else "No"
            each_row_dict['total_quizitems'] = analytics_data['total_quizitems']
            each_row_dict['visited_quizitems'] = analytics_data['visited_quizitems']
            each_row_dict['unattempted_quizitems'] = analytics_data['unattempted_quizitems']
            each_row_dict['attempted_quizitems'] = analytics_data['attempted_quizitems']
            each_row_dict['incorrect_attempted_quizitems'] = analytics_data['incorrect_attempted_quizitems']
            each_row_dict['correct_attempted_quizitems'] = analytics_data['correct_attempted_quizitems']
            each_row_dict['notapplicable_quizitems'] = analytics_data['notapplicable_quizitems']
            each_row_dict['unique_users_commented_on_user_files'] = analytics_data['unique_users_commented_on_user_files']
            each_row_dict['user_files'] = analytics_data['user_files']
            each_row_dict['total_files_viewed_by_user'] = analytics_data['total_files_viewed_by_user']
            each_row_dict['other_viewing_my_files'] = analytics_data['other_viewing_my_files']
            each_row_dict['total_rating_rcvd_on_files'] = analytics_data['total_rating_rcvd_on_files']
            each_row_dict['commented_on_others_files'] = analytics_data['commented_on_others_files']
            each_row_dict['cmts_on_user_files'] = analytics_data['cmts_on_user_files']
            each_row_dict['total_cmnts_by_user'] = analytics_data['total_cmnts_by_user']
            each_row_dict['user_notes'] = analytics_data['user_notes']
            each_row_dict['others_reading_my_notes'] = analytics_data['others_reading_my_notes']
            each_row_dict['cmts_on_user_notes'] = analytics_data['cmts_on_user_notes']
            each_row_dict['cmnts_rcvd_by_user'] = analytics_data['cmnts_rcvd_by_user']
            each_row_dict['total_notes_read_by_user'] = analytics_data['total_notes_read_by_user']
            each_row_dict['commented_on_others_notes'] = analytics_data['commented_on_others_notes']
            each_row_dict['total_rating_rcvd_on_notes'] = analytics_data['total_rating_rcvd_on_notes']
            each_row_dict["correct_attempted_assessments"] = analytics_data["correct_attempted_assessments"]
            each_row_dict["unattempted_assessments"] = analytics_data["unattempted_assessments"]
            each_row_dict["visited_assessments"] = analytics_data["visited_assessments"]
            each_row_dict["notapplicable_assessments"] = analytics_data["notapplicable_assessments"]
            each_row_dict["incorrect_attempted_assessments"] = analytics_data["incorrect_attempted_assessments"]
            each_row_dict["attempted_assessments"] = analytics_data["attempted_assessments"]
            each_row_dict["total_assessment_items"] = analytics_data["total_assessment_items"]
            # each_row_dict["buddies"] = str(Buddy.get_buddy_userids_list_within_datetime(1, datetime.datetime.now()))
            each_row_dict["buddies"] = Author.get_author_usernames_list_from_user_id_list\
                (Buddy.get_buddy_userids_list_within_datetime(each_user, datetime.datetime.now()))
            # print analytics_data

            with open(file_name_path, 'a') as f:  # Just use 'w' mode in 3.x
                w = csv.DictWriter(f, column_keys_list)
                if not header_written:
                    w.writeheader()
                    header_written = True
                w.writerow(each_row_dict)

        except Exception, e:
            print e
            error_msg = "\nUser with id: " + str(each_user) + " does not exists!!"
            log_file.write(str(e))
            log_file.write(error_msg)
            continue
            <commit_msg>User Analytics: added new columns for activities and their visit count<commit_after>''' -- imports from python libraries -- '''
import os
import time
import datetime
import csv
from collections import OrderedDict
# import json

''' imports from installed packages '''
from django.utils.text import slugify
from django.core.management.base import BaseCommand, CommandError
# from django.contrib.auth.models import User

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Node, Group, GSystemType, Counter, node_collection, Buddy, Author
from gnowsys_ndf.settings import GSTUDIO_LOGS_DIR_PATH, GSTUDIO_DATA_ROOT
from gnowsys_ndf.ndf.views.gcourse import course_analytics
# from gnowsys_ndf.ndf.views.methods import get_group_name_id
# from gnowsys_ndf.ndf.views.analytics_methods import *

try:
    from gnowsys_ndf.server_settings import GSTUDIO_INSTITUTE_ID, GSTUDIO_INSTITUTE_ID_SECONDARY, GSTUDIO_INSTITUTE_NAME
except Exception, e:
    from gnowsys_ndf.settings import GSTUDIO_INSTITUTE_ID, GSTUDIO_INSTITUTE_ID_SECONDARY, GSTUDIO_INSTITUTE_NAME

if not os.path.exists(GSTUDIO_LOGS_DIR_PATH):
    os.makedirs(GSTUDIO_LOGS_DIR_PATH)

log_file_name = 'export_users_analytics.log'
log_file_path = os.path.join(GSTUDIO_LOGS_DIR_PATH, log_file_name)
log_file = open(log_file_path, 'a+')
script_start_str = "\n\n######### Script ran on : " + time.strftime("%c") + " #########\n----------------\n"
log_file.write(str(script_start_str))

column_keys_list = ["server_id", "school_name", "school_code", "unit_name", "username", "user_id", "enrollment_status", "total_lessons", "lessons_completed", "percentage_lessons_completed", "total_activities", "activities_completed", "percentage_activities_completed", "total_quizitems", "visited_quizitems", "attempted_quizitems", "unattempted_quizitems", "correct_attempted_quizitems", "notapplicable_quizitems", "incorrect_attempted_quizitems", "user_files", "total_files_viewed_by_user", "other_viewing_my_files", "unique_users_commented_on_user_files", "total_rating_rcvd_on_files", "commented_on_others_files", "cmts_on_user_files", "total_cmnts_by_user", "user_notes", "others_reading_my_notes", "cmts_on_user_notes", "cmnts_rcvd_by_user", "total_notes_read_by_user", "commented_on_others_notes", "total_rating_rcvd_on_notes", "correct_attempted_assessments", "unattempted_assessments", "visited_assessments", "notapplicable_assessments", "incorrect_attempted_assessments", "attempted_assessments", "total_assessment_items", "buddies"]

column_keys_dict = OrderedDict()
map(lambda x: column_keys_dict.update({x: "NA"}), column_keys_list)

class Command(BaseCommand):

    def handle(self, *args, **options):
        gst_base_unit_name, gst_base_unit_id = GSystemType.get_gst_name_id('announced_unit')
        gst_course_ev_gr = node_collection.one({'_type': 'GSystemType', 'name': 'CourseEventGroup'})
        all_course_groups = node_collection.find({'_type': 'Group', 'member_of': {'$in': [gst_course_ev_gr._id, gst_base_unit_id]} })

        for each_group_obj in all_course_groups:
            assessment_and_quiz_data = True
            print "\n\n Exporting CSV for :", each_group_obj.name, "(Altnames:", each_group_obj.altnames, ")\n"
            export_group_analytics(each_group_obj, assessment_and_quiz_data)

def export_group_analytics(group_obj, assessment_and_quiz_data):

    # print "Enter 'group name(case sensitive)' OR 'id': "
    # group_name_or_id = raw_input()
    # group_obj = Group.get_group_name_id(group_name_or_id, get_obj=True)

    # if not group_obj:
    #     raise ValueError('\nSorry. Request could not be completed. \
    #         \nGroup/Course, matching argument entered "' + group_name_or_id + '" does not exists!')

    # group_users = group_obj.author_set
    # Previously, above was implementation strategy. Now we are using counter users instead of group's author_set,
    # because we do wanted to gather all data irrespective of student enrollment.
    group_users = Counter.get_group_counters(group_obj._id).distinct('user_id')
    # group_users is a list of integer user ids
    # print group_users

    # CSV file name-convention: schoolcode-course-name-datetimestamp.csv
    try:
        group_name = slugify(group_obj['name'])
    except Exception, e:
        print e
        group_name = str(group_obj['_id'])

    # dt: date time
    # e.g: '21-November-2016-19h-08m-10s'
    # dt = "{:%d-%B-%Y-%Hh-%Mm-%Ss}".format(datetime.datetime.now())
    dt = "{:%Y%m%d-%Hh%Mm}".format(datetime.datetime.now())

    file_name = GSTUDIO_INSTITUTE_ID_SECONDARY + '-' + GSTUDIO_INSTITUTE_ID + '-' + group_name + '-' + dt + '.csv'

    GSTUDIO_EXPORTED_CSVS_DIRNAME = 'gstudio-exported-users-analytics-csvs'
    GSTUDIO_EXPORTED_CSVS_DIR_PATH = os.path.join('/data/', GSTUDIO_EXPORTED_CSVS_DIRNAME)

    if not os.path.exists(GSTUDIO_EXPORTED_CSVS_DIR_PATH):
        os.makedirs(GSTUDIO_EXPORTED_CSVS_DIR_PATH)

    file_name_path = os.path.join(GSTUDIO_EXPORTED_CSVS_DIR_PATH, file_name)

    header_written = False
    for index, each_user in enumerate(group_users):
        try:
            analytics_data = course_analytics(None, group_obj._id, each_user, get_result_dict=True, assessment_and_quiz_data=assessment_and_quiz_data, get_counter_obj_in_result=True)
            if not analytics_data:
                continue

            print index, "] Group User ID: ", each_user

            each_row_dict = column_keys_dict.copy()
            each_row_dict['server_id'] = GSTUDIO_INSTITUTE_ID
            each_row_dict['school_code'] = GSTUDIO_INSTITUTE_ID_SECONDARY
            each_row_dict['school_name'] = GSTUDIO_INSTITUTE_NAME
            each_row_dict['unit_name'] = group_name

            temp_lessons_stat_str = analytics_data['level1_progress_stmt']
            temp_lessons_stat_str = analytics_data['level1_progress_stmt']
            each_row_dict['lessons_completed'] = int(temp_lessons_stat_str.split(' ')[0])
            each_row_dict['total_lessons'] = int(temp_lessons_stat_str.split(' ')[3])

            temp_activities_stat_str = analytics_data['level2_progress_stmt']
            each_row_dict['activities_completed'] = int(temp_activities_stat_str.split(' ')[0])
            each_row_dict['total_activities'] = int(temp_activities_stat_str.split(' ')[3])

            each_row_dict['percentage_lessons_completed'] = analytics_data['level1_progress_meter']
            each_row_dict['percentage_activities_completed'] = analytics_data['level2_progress_meter']

            each_row_dict['username'] = analytics_data['username']
            each_row_dict['user_id'] = each_user
            each_row_dict['enrollment_status'] = "Yes" if (each_user in group_obj.author_set) else "No"
            each_row_dict['total_quizitems'] = analytics_data['total_quizitems']
            each_row_dict['visited_quizitems'] = analytics_data['visited_quizitems']
            each_row_dict['unattempted_quizitems'] = analytics_data['unattempted_quizitems']
            each_row_dict['attempted_quizitems'] = analytics_data['attempted_quizitems']
            each_row_dict['incorrect_attempted_quizitems'] = analytics_data['incorrect_attempted_quizitems']
            each_row_dict['correct_attempted_quizitems'] = analytics_data['correct_attempted_quizitems']
            each_row_dict['notapplicable_quizitems'] = analytics_data['notapplicable_quizitems']
            each_row_dict['unique_users_commented_on_user_files'] = analytics_data['unique_users_commented_on_user_files']
            each_row_dict['user_files'] = analytics_data['user_files']
            each_row_dict['total_files_viewed_by_user'] = analytics_data['total_files_viewed_by_user']
            each_row_dict['other_viewing_my_files'] = analytics_data['other_viewing_my_files']
            each_row_dict['total_rating_rcvd_on_files'] = analytics_data['total_rating_rcvd_on_files']
            each_row_dict['commented_on_others_files'] = analytics_data['commented_on_others_files']
            each_row_dict['cmts_on_user_files'] = analytics_data['cmts_on_user_files']
            each_row_dict['total_cmnts_by_user'] = analytics_data['total_cmnts_by_user']
            each_row_dict['user_notes'] = analytics_data['user_notes']
            each_row_dict['others_reading_my_notes'] = analytics_data['others_reading_my_notes']
            each_row_dict['cmts_on_user_notes'] = analytics_data['cmts_on_user_notes']
            each_row_dict['cmnts_rcvd_by_user'] = analytics_data['cmnts_rcvd_by_user']
            each_row_dict['total_notes_read_by_user'] = analytics_data['total_notes_read_by_user']
            each_row_dict['commented_on_others_notes'] = analytics_data['commented_on_others_notes']
            each_row_dict['total_rating_rcvd_on_notes'] = analytics_data['total_rating_rcvd_on_notes']
            each_row_dict["correct_attempted_assessments"] = analytics_data["correct_attempted_assessments"]
            each_row_dict["unattempted_assessments"] = analytics_data["unattempted_assessments"]
            each_row_dict["visited_assessments"] = analytics_data["visited_assessments"]
            each_row_dict["notapplicable_assessments"] = analytics_data["notapplicable_assessments"]
            each_row_dict["incorrect_attempted_assessments"] = analytics_data["incorrect_attempted_assessments"]
            each_row_dict["attempted_assessments"] = analytics_data["attempted_assessments"]
            each_row_dict["total_assessment_items"] = analytics_data["total_assessment_items"]
            # each_row_dict["buddies"] = str(Buddy.get_buddy_userids_list_within_datetime(1, datetime.datetime.now()))
            each_row_dict["buddies"] = Author.get_author_usernames_list_from_user_id_list\
                (Buddy.get_buddy_userids_list_within_datetime(each_user, datetime.datetime.now()))
            all_activities_cur = Node.get_tree_nodes(group_obj, field_name='collection_set', level=1, get_obj=True)
            for each_act in all_activities_cur:
                column_key_name = each_act['altnames'] if each_act['altnames'].strip() else each_act['name']
                column_key_name += " [" + unicode(each_act._id) + "]"
                if column_key_name not in column_keys_list:
                    column_keys_list.append(column_key_name)
                    column_keys_dict.update({column_key_name: 0})
                each_row_dict[column_key_name] = analytics_data["counter_obj"]["visited_nodes"].get(unicode(each_act._id), 0)
            # print analytics_data

            with open(file_name_path, 'a') as f:  # Just use 'w' mode in 3.x
                w = csv.DictWriter(f, column_keys_list)
                if not header_written:
                    w.writeheader()
                    header_written = True
                w.writerow(each_row_dict)

        except Exception, e:
            print e
            error_msg = "\nUser with id: " + str(each_user) + " does not exists!!"
            log_file.write(str(e))
            log_file.write(error_msg)
            continue

    print "\nExported user CSV: %s"%file_name_path
            <|endoftext|>


<commit_before>from hbpush.message import Message
import time

class ChannelRegistry(object):
    def __init__(self, channel_cls, store):
        self.channel_cls = channel_cls
        self.store = store

    def get(self, id, callback, errback):
        raise NotImplementedError("")

    def create(self, id, callback, errback, overwrite=False):
        raise NotImplementedError("")

    def get_or_create(self, id, callback, errback):
        raise NotImplementedError("")

    def delete(self, id, callback, errback):
        raise NotImplementedError("")

class Channel(object):
    def __init__(self, id, store):
        self.store = store
        self.id = id

    def post(self, message, callback):
        raise NotImplementedError("")

    def get(self, last_modified, etag, callback):
        raise NotImplementedError("")

    def get_last_message(self):
        raise NotImplementedError("")

    def subscribe(self, id_subscriber, callback):
        raise NotImplementedError("")

    def unsubscribe(self, id_subscriber):
        raise NotImplementedError("")

    def make_message(self, content_type, body):
        last_message = self.get_last_message()

        last_modified = int(time.time())
        if last_modified == last_message.last_modified:
            etag = last_message.etag+1
        else:
            etag = 0

        return Message(last_modified, etag, content_type, body)

    class DoesNotExist(Exception):
        pass

    class Duplicate(Exception):
        pass

    class Gone(Exception):
        pass

    class NotModified(Exception):
        pass
<commit_msg>bleeh, that message etag generation was incorrect with asynchronous posting<commit_after>from hbpush.message import Message
import time

class ChannelRegistry(object):
    def __init__(self, channel_cls, store):
        self.channel_cls = channel_cls
        self.store = store

    def get(self, id, callback, errback):
        raise NotImplementedError("")

    def create(self, id, callback, errback, overwrite=False):
        raise NotImplementedError("")

    def get_or_create(self, id, callback, errback):
        raise NotImplementedError("")

    def delete(self, id, callback, errback):
        raise NotImplementedError("")

class Channel(object):
    def __init__(self, id, store):
        self.store = store
        self.id = id
        self.sentinel = None

    def post(self, message, callback):
        raise NotImplementedError("")

    def get(self, last_modified, etag, callback):
        raise NotImplementedError("")

    def get_last_message(self):
        raise NotImplementedError("")

    def subscribe(self, id_subscriber, callback):
        raise NotImplementedError("")

    def unsubscribe(self, id_subscriber):
        raise NotImplementedError("")

    def make_message(self, content_type, body):
        if not self.sentinel:
            self.sentinel = self.get_last_message()

        last_modified = int(time.time())
        if last_modified == self.sentinel.last_modified:
            etag = self.sentinel.etag+1
        else:
            etag = 0

        self.sentinel = Message(last_modified, etag, content_type, body)
        return self.sentinel

    class DoesNotExist(Exception):
        pass

    class Duplicate(Exception):
        pass

    class Gone(Exception):
        pass

    class NotModified(Exception):
        pass
<|endoftext|>


<commit_before>"""HDL Code generation."""

from .stmt import HDLStatement

def indent(fn):
    """Indent decorator."""
    def wrapper(*args):
        if not isinstance(args[0], HDLCodeGenerator):
            raise TypeError('decorator can only be used on HDLCodeGenerator '
                            'objects')
        # generate indentation
        args[0].increase_indent()
        if args[0].indent is True:
            indent_str = args[0].indent_str
        else:
            indent_str = ''

        print(args[0].indent_level)
        ret = '\n'.join([indent_str + x for x in fn(*args).split('\n')])
        args[0].decrease_indent()
        return ret
    return wrapper

class HDLCodeGenerator(object):
    """Abstract class for code generators."""

    statements = []
    class_aliases = {}

    def __init__(self, indent=False, indent_str='    '):
        """Initialize."""
        self.indent_level = 0
        self.indent = indent
        self.indent_str = indent_str

    def increase_indent(self):
        """Increase indentation level."""
        self.indent_level += 1

    def decrease_indent(self):
        """Decrease indentation level."""
        if self.indent_level > 0:
            self.indent_level -= 1

    def add_class_alias(self, use_as, alias_class):
        """Add class alias."""
        self.class_aliases[alias_class] = use_as

    def dump_element(self, element, **kwargs):
        """Get code representation for an element."""
        cls_name = element.__class__.__name__
        gen_method_name = 'gen_{}'.format(cls_name)

        # check statement validity
        if isinstance(element, HDLStatement):
            if element.is_legal() is False:
                raise ValueError('illegal statement passed')

        # aliases
        if cls_name in self.class_aliases:
            cls_name = self.class_aliases[cls_name]
            gen_method_name = 'gen_{}'.format(cls_name)

        try:
            return getattr(self, gen_method_name)(element, **kwargs)
        except AttributeError:
            raise TypeError('cannot generate code for object type '
                            '"{}"'.format(cls_name))
    # basic types to be implemented

    def gen_HDLIntegerConstant(self, element, **kwargs):
        """Generate Integer constants."""
        raise NotImplementedError

    def gen_HDLStringConstant(self, element, **kwargs):
        """Generate String constants."""
        raise NotImplementedError

    def gen_HDLVectorDescriptor(self, element, **kwargs):
        """Generate Vector elements."""
        raise NotImplementedError

    def gen_HDLModulePort(self, element, **kwargs):
        """Generate ports."""
        raise NotImplementedError

    # builtin types

    def gen_int(self, element, **kwargs):
        """Integer."""
        return str(element)

    def get_str(self, element, **kwargs):
        """String."""
        return element
<commit_msg>remove debug garbage<commit_after>"""HDL Code generation."""

from .stmt import HDLStatement

def indent(fn):
    """Indent decorator."""
    def wrapper(*args):
        if not isinstance(args[0], HDLCodeGenerator):
            raise TypeError('decorator can only be used on HDLCodeGenerator '
                            'objects')
        # generate indentation
        args[0].increase_indent()
        if args[0].indent is True:
            indent_str = args[0].indent_str
        else:
            indent_str = ''

        ret = '\n'.join([indent_str + x for x in fn(*args).split('\n')])
        args[0].decrease_indent()
        return ret
    return wrapper

class HDLCodeGenerator(object):
    """Abstract class for code generators."""

    statements = []
    class_aliases = {}

    def __init__(self, indent=False, indent_str='    '):
        """Initialize."""
        self.indent_level = 0
        self.indent = indent
        self.indent_str = indent_str

    def increase_indent(self):
        """Increase indentation level."""
        self.indent_level += 1

    def decrease_indent(self):
        """Decrease indentation level."""
        if self.indent_level > 0:
            self.indent_level -= 1

    def add_class_alias(self, use_as, alias_class):
        """Add class alias."""
        self.class_aliases[alias_class] = use_as

    def dump_element(self, element, **kwargs):
        """Get code representation for an element."""
        cls_name = element.__class__.__name__
        gen_method_name = 'gen_{}'.format(cls_name)

        # check statement validity
        if isinstance(element, HDLStatement):
            if element.is_legal() is False:
                raise ValueError('illegal statement passed')

        # aliases
        if cls_name in self.class_aliases:
            cls_name = self.class_aliases[cls_name]
            gen_method_name = 'gen_{}'.format(cls_name)

        try:
            return getattr(self, gen_method_name)(element, **kwargs)
        except AttributeError:
            raise TypeError('cannot generate code for object type '
                            '"{}"'.format(cls_name))
    # basic types to be implemented

    def gen_HDLIntegerConstant(self, element, **kwargs):
        """Generate Integer constants."""
        raise NotImplementedError

    def gen_HDLStringConstant(self, element, **kwargs):
        """Generate String constants."""
        raise NotImplementedError

    def gen_HDLVectorDescriptor(self, element, **kwargs):
        """Generate Vector elements."""
        raise NotImplementedError

    def gen_HDLModulePort(self, element, **kwargs):
        """Generate ports."""
        raise NotImplementedError

    # builtin types

    def gen_int(self, element, **kwargs):
        """Integer."""
        return str(element)

    def get_str(self, element, **kwargs):
        """String."""
        return element
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
_configuration_

Util class to read and access the cirrus.conf file
that defines how the various commands operate.

Commands should access this by doing

from cirrus.configuration import load_configuration
conf = load_configuration()

"""
import os
import gitconfig
import subprocess
import ConfigParser

class Configuration(dict):
    """
    _Configuration_

    """
    def __init__(self, config_file):
        super(Configuration, self).__init__(self)
        self.config_file = config_file
        self.parser = None

    def load(self):
        """
        _load_from_file_

        Reread the cirrus config file

        """
        self.parser = ConfigParser.RawConfigParser()
        self.parser.read(self.config_file)
        for section in self.parser.sections():
            self.setdefault(section, {})
            for option in self.parser.options(section):
                self[section].setdefault(
                    option,
                    self.parser.get(section, option)
                )

    def has_section(self, section):
        return section in self

    def get_param(self, section, param, default=None):
        """
        _get_param_

        convienience param getter with section, param name and
        optional default to avoid key errors
        """
        if section not in self:
            raise KeyError('section {0} not found'.format(section))
        return self[section].get(param, default)

    def package_version(self):
        return self.get('package', {}).get('version')

    def package_name(self):
        return self.get('package', {}).get('name')

    def organisation_name(self):
        return self.get('package', {}).get('organization')

    def release_notes_format(self):
        return self.get('package', {}).get('release_notes_format', 'plaintext')

    def pypi_url(self):
        return self.get('pypi', {}).get('pypi_url')

    def pypi_config(self):
        """
        get the details for uploading to pypi

        """
        return self.get('pypi', {})

    def gitflow_branch_name(self):
        return self.get('gitflow', {}).get('develop_branch', 'develop')

    def gitflow_master_name(self):
        return self.get('gitflow', {}).get('master_branch', 'master')

    def gitflow_feature_prefix(self):
        return self.get('gitflow', {}).get('feature_branch_prefix', 'feature/')

    def gitflow_release_prefix(self):
        return self.get('gitflow', {}).get('release_branch_prefix', 'release/')

    def test_where(self, suite):
        return self.get('test-{0}'.format(suite), {}).get('where')

    def venv_name(self):
        return self.get('build', {}).get('virtualenv_name', 'venv')

    def quality_rcfile(self):
        return self.get('quality', {}).get('rcfile')

    def quality_threshold(self):
        return float(self.get('quality', {}).get('threshold'))

    def release_notes(self):
        """
        returns the release notes file and release
        notes sentinel from the config
        """
        return (
            self.get('package', {}).get('release_notes_file'),
            self.get('package', {}).get('release_notes_sentinel')
        )

    def version_file(self):
        """
        returns the version file and version attr from the configuration
        """
        return (
            self.get('package', {}).get('version_file'),
            self.get('package', {}).get('version_attribute', '__version__')
        )

    def update_package_version(self, new_version):
        """
        _update_package_version_

        Update the version in the configuration field
        """
        self['package']['version'] = new_version
        self.parser.set('package', 'version', new_version)
        with open(self.config_file, 'w') as handle:
            self.parser.write(handle)

def _repo_directory():
    command = ['git', 'rev-parse', '--show-toplevel']
    process = subprocess.Popen([command], stdout=subprocess.PIPE)
    outp, err = process.communicate()
    return outp.strip()

def load_configuration(package_dir=None):
    """
    _load_configuration_

    Load the cirrus.conf file and parse it into a nested dictionary
    like Configuration instance.

    :params package_dir: Location of cirrus managed package if not pwd
    :returns: Configuration instance

    """
    dirname = os.getcwd()
    if package_dir is not None:
        dirname = package_dir

    config_path = os.path.join(dirname, 'cirrus.conf')
    if not os.path.exists(config_path):
        repo_dir = _repo_directory()
        config_path = os.path.join(repo_dir, 'cirrus.conf')

    if not os.path.exists(config_path):
        msg = "Couldnt find ./cirrus.conf, are you in a package directory?"
        raise RuntimeError(msg)

    config_instance = Configuration(config_path)
    config_instance.load()
    return config_instance

def get_github_auth():
    """
    _get_git_auth_

    Pull in github auth user & token from gitconfig
    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    github_user = config.get('cirrus', 'github-user')
    github_token = config.get('cirrus', 'github-token')
    return github_user, github_token

def get_pypi_auth():
    """
    _pypi_auth_

    Get pypi credentials from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    pypi_user = config.get('cirrus', 'pypi-user')
    pypi_ssh_user = config.get('cirrus', 'pypi-ssh-user')
    pypi_key = config.get('cirrus', 'pypi-ssh-key')
    pypi_token = config.get('cirrus', 'pypi-token')
    return {
        'username': pypi_user,
        'ssh_username': pypi_ssh_user,
        'ssh_key': pypi_key,
        'token': pypi_token
    }

def get_buildserver_auth():
    """
    _buildserver_auth_

    Get buildserver credentials from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    return (config.get('cirrus', 'buildserver-user'),
            config.get('cirrus', 'buildserver-token'),)

def get_chef_auth():
    """
    get chef auth info from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    return {
        'chef_server': config.get('cirrus', 'chef-server'),
        'chef_username': config.get('cirrus', 'chef-username'),
        'chef_keyfile': config.get('cirrus', 'chef-keyfile'),
        'chef_client_user': config.get('cirrus', 'chef-client-user'),
        'chef_client_keyfile': config.get('cirrus', 'chef-client-keyfile')
    }

if __name__ == '__main__':
    load_configuration()
<commit_msg>bugfix double [] around command<commit_after>#!/usr/bin/env python
"""
_configuration_

Util class to read and access the cirrus.conf file
that defines how the various commands operate.

Commands should access this by doing

from cirrus.configuration import load_configuration
conf = load_configuration()

"""
import os
import gitconfig
import subprocess
import ConfigParser

class Configuration(dict):
    """
    _Configuration_

    """
    def __init__(self, config_file):
        super(Configuration, self).__init__(self)
        self.config_file = config_file
        self.parser = None

    def load(self):
        """
        _load_from_file_

        Reread the cirrus config file

        """
        self.parser = ConfigParser.RawConfigParser()
        self.parser.read(self.config_file)
        for section in self.parser.sections():
            self.setdefault(section, {})
            for option in self.parser.options(section):
                self[section].setdefault(
                    option,
                    self.parser.get(section, option)
                )

    def has_section(self, section):
        return section in self

    def get_param(self, section, param, default=None):
        """
        _get_param_

        convienience param getter with section, param name and
        optional default to avoid key errors
        """
        if section not in self:
            raise KeyError('section {0} not found'.format(section))
        return self[section].get(param, default)

    def package_version(self):
        return self.get('package', {}).get('version')

    def package_name(self):
        return self.get('package', {}).get('name')

    def organisation_name(self):
        return self.get('package', {}).get('organization')

    def release_notes_format(self):
        return self.get('package', {}).get('release_notes_format', 'plaintext')

    def pypi_url(self):
        return self.get('pypi', {}).get('pypi_url')

    def pypi_config(self):
        """
        get the details for uploading to pypi

        """
        return self.get('pypi', {})

    def gitflow_branch_name(self):
        return self.get('gitflow', {}).get('develop_branch', 'develop')

    def gitflow_master_name(self):
        return self.get('gitflow', {}).get('master_branch', 'master')

    def gitflow_feature_prefix(self):
        return self.get('gitflow', {}).get('feature_branch_prefix', 'feature/')

    def gitflow_release_prefix(self):
        return self.get('gitflow', {}).get('release_branch_prefix', 'release/')

    def test_where(self, suite):
        return self.get('test-{0}'.format(suite), {}).get('where')

    def venv_name(self):
        return self.get('build', {}).get('virtualenv_name', 'venv')

    def quality_rcfile(self):
        return self.get('quality', {}).get('rcfile')

    def quality_threshold(self):
        return float(self.get('quality', {}).get('threshold'))

    def release_notes(self):
        """
        returns the release notes file and release
        notes sentinel from the config
        """
        return (
            self.get('package', {}).get('release_notes_file'),
            self.get('package', {}).get('release_notes_sentinel')
        )

    def version_file(self):
        """
        returns the version file and version attr from the configuration
        """
        return (
            self.get('package', {}).get('version_file'),
            self.get('package', {}).get('version_attribute', '__version__')
        )

    def update_package_version(self, new_version):
        """
        _update_package_version_

        Update the version in the configuration field
        """
        self['package']['version'] = new_version
        self.parser.set('package', 'version', new_version)
        with open(self.config_file, 'w') as handle:
            self.parser.write(handle)

def _repo_directory():
    command = ['git', 'rev-parse', '--show-toplevel']
    process = subprocess.Popen(command, stdout=subprocess.PIPE)
    outp, err = process.communicate()
    return outp.strip()

def load_configuration(package_dir=None):
    """
    _load_configuration_

    Load the cirrus.conf file and parse it into a nested dictionary
    like Configuration instance.

    :params package_dir: Location of cirrus managed package if not pwd
    :returns: Configuration instance

    """
    dirname = os.getcwd()
    if package_dir is not None:
        dirname = package_dir

    config_path = os.path.join(dirname, 'cirrus.conf')
    if not os.path.exists(config_path):
        repo_dir = _repo_directory()
        config_path = os.path.join(repo_dir, 'cirrus.conf')

    if not os.path.exists(config_path):
        msg = "Couldnt find ./cirrus.conf, are you in a package directory?"
        raise RuntimeError(msg)

    config_instance = Configuration(config_path)
    config_instance.load()
    return config_instance

def get_github_auth():
    """
    _get_git_auth_

    Pull in github auth user & token from gitconfig
    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    github_user = config.get('cirrus', 'github-user')
    github_token = config.get('cirrus', 'github-token')
    return github_user, github_token

def get_pypi_auth():
    """
    _pypi_auth_

    Get pypi credentials from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    pypi_user = config.get('cirrus', 'pypi-user')
    pypi_ssh_user = config.get('cirrus', 'pypi-ssh-user')
    pypi_key = config.get('cirrus', 'pypi-ssh-key')
    pypi_token = config.get('cirrus', 'pypi-token')
    return {
        'username': pypi_user,
        'ssh_username': pypi_ssh_user,
        'ssh_key': pypi_key,
        'token': pypi_token
    }

def get_buildserver_auth():
    """
    _buildserver_auth_

    Get buildserver credentials from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    return (config.get('cirrus', 'buildserver-user'),
            config.get('cirrus', 'buildserver-token'),)

def get_chef_auth():
    """
    get chef auth info from gitconfig

    """
    gitconfig_file = os.path.join(os.environ['HOME'], '.gitconfig')
    config = gitconfig.config(gitconfig_file)
    return {
        'chef_server': config.get('cirrus', 'chef-server'),
        'chef_username': config.get('cirrus', 'chef-username'),
        'chef_keyfile': config.get('cirrus', 'chef-keyfile'),
        'chef_client_user': config.get('cirrus', 'chef-client-user'),
        'chef_client_keyfile': config.get('cirrus', 'chef-client-keyfile')
    }

if __name__ == '__main__':
    load_configuration()
<|endoftext|>


<commit_before>import os

from data.Track.UploadHandler import UploadHandler

from src.data.Track.Tracks import YoutubeTrack

class YoutubeUploadHandler(UploadHandler):

    def __init__(self, workingDir):
        super().__init__(workingDir)

        self.attributes.update({
            "URL": ["string", "required", "url"]
        })

    def trackFromUploadedAttributes(self, attributes):

        track = YoutubeTrack(
            attributes["Artist"],
            attributes["Album"],
            attributes["Title"]
        )

        del attributes["Artist"]
        del attributes["Album"]
        del attributes["Title"]

        super().autoImportAttributes(track, attributes)
        super().writeTrackRecord(track)

        artistPath = os.path.join(self.workingDir, track.artistName)
        albumPath = os.path.join(artistPath, track.albumTitle)
        recordPath = os.path.join(albumPath, track.title) + ".rec"
        localFilePath = os.path.join(recordPath, "muzak.yturl")

        fileToWrite = open(localFilePath, 'w+')
        fileToWrite.write(track.url)
        fileToWrite.close()

        return track
<commit_msg>Fix wrong import from UploadHandler<commit_after>import os

from src.data.Track import UploadHandler
from src.data.Track.Tracks import YoutubeTrack

class YoutubeUploadHandler(UploadHandler):

    def __init__(self, workingDir):
        super().__init__(workingDir)

        self.attributes.update({
            "URL": ["string", "required", "url"]
        })

    def trackFromUploadedAttributes(self, attributes):

        track = YoutubeTrack(
            attributes["Artist"],
            attributes["Album"],
            attributes["Title"]
        )

        del attributes["Artist"]
        del attributes["Album"]
        del attributes["Title"]

        super().autoImportAttributes(track, attributes)
        super().writeTrackRecord(track)

        artistPath = os.path.join(self.workingDir, track.artistName)
        albumPath = os.path.join(artistPath, track.albumTitle)
        recordPath = os.path.join(albumPath, track.title) + ".rec"
        localFilePath = os.path.join(recordPath, "muzak.yturl")

        fileToWrite = open(localFilePath, 'w+')
        fileToWrite.write(track.url)
        fileToWrite.close()

        return track
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

from qa_utils.rest_client_utils import RestClient, API_ROOT_URL_ARG_NAME, model_to_request_body,  \
    response_body_to_dict, HEADER_CONTENT_TYPE, HEADER_REPRESENTATION_XML
from qa_utils.logger_utils import get_logger

logger = get_logger("paasmanagerClient")

#URI ELEMENT
PAASMANAGER_BASE_URI = "{" + API_ROOT_URL_ARG_NAME + "}"
ENVIRONMENT_RESOURCE_ROOT_URI = PAASMANAGER_BASE_URI + "/catalog/org/FIWARE/vdc/{tenant_id}/environment"
ENVIRONMENT_RESOURCE_DETAIL_URI = ENVIRONMENT_RESOURCE_ROOT_URI + "/{environment_name}"

# BODY ELEMENTS
ENVIRONMENT_BODY_ROOT = "environmentDto"
ENVIRONMENT_BODY_NAME = "name"
ENVIRONMENT_BODY_DESCRIPTION = "description"

class EnvironmentResourceClient(RestClient):

    def __init__(self, protocol, host, port, tenant_id, resource=None, headers=None):
        """
        Class constructor. Inits default attributes.
        :param protocol: Connection protocol (HTTP | HTTPS)
        :param host: Host
        :param port: Port
        :param tenant_id: TenantID
        :param resource: Base URI resource
        :param headers: HTTP Headers
        :return: None
        """
        if headers is None:
            self.headers = {HEADER_CONTENT_TYPE: HEADER_REPRESENTATION_XML}
        self.headers = headers
        self.tenant_id = tenant_id
        super(EnvironmentResourceClient, self).__init__(protocol, host, port, resource=resource)

    def create_environment(self, name, description=None):
        """
        Create a new environment (Tenant)
        :param name: Name of the environment
        :param description: Description of the environment
        :return: 'Requests' response
        """
        logger.info("Creating new environment")
        env_model = {ENVIRONMENT_BODY_ROOT: {ENVIRONMENT_BODY_NAME: name,
                                             ENVIRONMENT_BODY_DESCRIPTION: description}}
        body = model_to_request_body(env_model, self.headers[HEADER_CONTENT_TYPE])
        return self.post(ENVIRONMENT_RESOURCE_ROOT_URI, body, self.headers, parameters=None, tenant_id=self.tenant_id)

    def delete_environment(self, name):
        """
        Delete an environemnt (Tenant)
        :param name: Name of the environment to be deleted
        :return: 'Request' response
        """
        logger.info("Deleting environment")
        return self.delete(ENVIRONMENT_RESOURCE_DETAIL_URI, headers=self.headers, parameters=None,
                           tenant_id=self.tenant_id, environment_name=name)

def get_environment(self, name):
        """
        Get an environment (Tenant)
        :return:
        """
        logger.info("Get environment")
        response = self.get(ENVIRONMENT_RESOURCE_DETAIL_URI, headers=self.headers, parameters=None,
                           tenant_id=self.tenant_id, environment_name=name)

        sr_response = response_body_to_dict(response, self.headers[HEADER_CONTENT_TYPE],
                                          xml_root_element_name=ENVIRONMENT_BODY_ROOT)
        return sr_response<commit_msg>Change privacy of get_environment<commit_after># -*- coding: utf-8 -*-

# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

from qa_utils.rest_client_utils import RestClient, API_ROOT_URL_ARG_NAME, model_to_request_body,  \
    response_body_to_dict, HEADER_CONTENT_TYPE, HEADER_REPRESENTATION_XML
from qa_utils.logger_utils import get_logger

logger = get_logger("paasmanagerClient")

#URI ELEMENT
PAASMANAGER_BASE_URI = "{" + API_ROOT_URL_ARG_NAME + "}"
ENVIRONMENT_RESOURCE_ROOT_URI = PAASMANAGER_BASE_URI + "/catalog/org/FIWARE/vdc/{tenant_id}/environment"
ENVIRONMENT_RESOURCE_DETAIL_URI = ENVIRONMENT_RESOURCE_ROOT_URI + "/{environment_name}"

# BODY ELEMENTS
ENVIRONMENT_BODY_ROOT = "environmentDto"
ENVIRONMENT_BODY_NAME = "name"
ENVIRONMENT_BODY_DESCRIPTION = "description"

class EnvironmentResourceClient(RestClient):

    def __init__(self, protocol, host, port, tenant_id, resource=None, headers=None):
        """
        Class constructor. Inits default attributes.
        :param protocol: Connection protocol (HTTP | HTTPS)
        :param host: Host
        :param port: Port
        :param tenant_id: TenantID
        :param resource: Base URI resource
        :param headers: HTTP Headers
        :return: None
        """
        if headers is None:
            self.headers = {HEADER_CONTENT_TYPE: HEADER_REPRESENTATION_XML}
        self.headers = headers
        self.tenant_id = tenant_id
        super(EnvironmentResourceClient, self).__init__(protocol, host, port, resource=resource)

    def create_environment(self, name, description=None):
        """
        Create a new environment (Tenant)
        :param name: Name of the environment
        :param description: Description of the environment
        :return: 'Requests' response
        """
        logger.info("Creating new environment")
        env_model = {ENVIRONMENT_BODY_ROOT: {ENVIRONMENT_BODY_NAME: name,
                                             ENVIRONMENT_BODY_DESCRIPTION: description}}
        body = model_to_request_body(env_model, self.headers[HEADER_CONTENT_TYPE])
        return self.post(ENVIRONMENT_RESOURCE_ROOT_URI, body, self.headers, parameters=None, tenant_id=self.tenant_id)

    def delete_environment(self, name):
        """
        Delete an environemnt (Tenant)
        :param name: Name of the environment to be deleted
        :return: 'Request' response
        """
        logger.info("Deleting environment")
        return self.delete(ENVIRONMENT_RESOURCE_DETAIL_URI, headers=self.headers, parameters=None,
                           tenant_id=self.tenant_id, environment_name=name)

    def get_environment(self, name):
        """
        Get an environment (Tenant)
        :return:
        """
        logger.info("Get environment")
        response = self.get(ENVIRONMENT_RESOURCE_DETAIL_URI, headers=self.headers, parameters=None,
                           tenant_id=self.tenant_id, environment_name=name)

        sr_response = response_body_to_dict(response, self.headers[HEADER_CONTENT_TYPE],
                                          xml_root_element_name=ENVIRONMENT_BODY_ROOT)
        return sr_response<|endoftext|>


<commit_before># Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from pprint import pprint
from typing import Any, Dict, Iterable, Mapping, Optional, Union

import torch

import pytorch_lightning as pl
from pytorch_lightning.core import memory
from pytorch_lightning.loggers import LightningLoggerBase, LoggerCollection, TensorBoardLogger
from pytorch_lightning.trainer.connectors.logger_connector.result import _METRIC, MetricSource
from pytorch_lightning.trainer.states import RunningStage, TrainerFn
from pytorch_lightning.utilities import DeviceType
from pytorch_lightning.utilities.metrics import metrics_to_scalars
from pytorch_lightning.utilities.types import _EVALUATE_OUTPUT

class LoggerConnector:

    def __init__(self, trainer: 'pl.Trainer', log_gpu_memory: Optional[str] = None) -> None:
        self.trainer = trainer
        self.log_gpu_memory = log_gpu_memory
        self.eval_loop_results = []
        self._val_log_step: int = 0
        self._test_log_step: int = 0
        self._progress_bar_metrics: Dict[str, float] = {}
        self._logged_metrics: Dict[str, _METRIC] = {}
        self._callback_metrics: Dict[str, _METRIC] = {}
        self._gpus_metrics: Dict[str, str] = {}
        self._epoch_end_reached = False
        self._current_fx: Optional[str] = None
        self._batch_idx: Optional[int] = None
        self._split_idx: Optional[int] = None

    def on_trainer_init(
        self,
        logger: LightningLoggerBase,
        flush_logs_every_n_steps: int,
        log_every_n_steps: int,
        move_metrics_to_cpu: bool,
    ) -> None:
        self.configure_logger(logger)
        self.trainer.flush_logs_every_n_steps = flush_logs_every_n_steps
        self.trainer.log_every_n_steps = log_every_n_steps
        self.trainer.move_metrics_to_cpu = move_metrics_to_cpu

    @property
    def should_flush_logs(self) -> bool:
        should_flush = (self.trainer.global_step + 1) % self.trainer.flush_logs_every_n_steps == 0
        return should_flush or self.trainer.should_stop

    @property
    def should_update_logs(self) -> bool:
        should_log_every_n_steps = (self.trainer.global_step + 1) % self.trainer.log_every_n_steps == 0
        return should_log_every_n_steps or self.trainer.should_stop

    def configure_logger(self, logger: Union[bool, Iterable, LightningLoggerBase]) -> None:
        if logger is True:
            version = os.environ.get('PL_EXP_VERSION', self.trainer.slurm_job_id)

            # default logger
            self.trainer.logger = TensorBoardLogger(
                save_dir=self.trainer.default_root_dir, version=version, name='lightning_logs'
            )
        elif logger is False:
            self.trainer.logger = None
        else:
            if isinstance(logger, Iterable):
                self.trainer.logger = LoggerCollection(logger)
            else:
                self.trainer.logger = logger

    def log_metrics(self, metrics: Dict[str, _METRIC], step: Optional[int] = None) -> None:
        """Logs the metric dict passed in.
        If `step` parameter is None and `step` key is presented is metrics,
        uses metrics["step"] as a step

        Args:
            metrics: Metric values
            step: Step for which metrics should be logged. Default value is `self.global_step` during training or
                the total validation / test log step count during validation and testing.
        """
        if self.trainer.logger is None or not metrics:
            return

        # turn all tensors to scalars
        scalar_metrics = metrics_to_scalars(metrics)

        if step is None:
            step = scalar_metrics.pop("step", None)
        if step is None:
            # added metrics for convenience
            scalar_metrics.setdefault("epoch", self.trainer.current_epoch)
            step = self.trainer.global_step

        # log actual metrics
        if self.trainer.is_global_zero:
            self.trainer.logger.agg_and_log_metrics(scalar_metrics, step=step)
            self.trainer.logger.save()

        self._logged_metrics.update(scalar_metrics)

    """
    Evaluation metric updates
    """

    @property
    def _eval_log_step(self) -> Optional[int]:
        if self.trainer.state.stage is RunningStage.VALIDATING:
            return self._val_log_step
        if self.trainer.state.stage is RunningStage.TESTING:
            return self._test_log_step
        return None

    def _increment_eval_log_step(self) -> None:
        if self.trainer.state.stage is RunningStage.VALIDATING:
            self._val_log_step += 1
        elif self.trainer.state.stage is RunningStage.TESTING:
            self._test_log_step += 1

    def on_evaluation_batch_start(self, batch: Any, batch_idx: int, dataloader_idx: int, num_dataloaders: int) -> None:
        model = self.trainer.lightning_module
        # set dataloader_idx only if multiple ones
        model._current_dataloader_idx = dataloader_idx if num_dataloaders > 1 else None

        # track batch_size
        self.trainer._results.extract_batch_size(batch)
        self._batch_idx = batch_idx

    def update_eval_step_metrics(self) -> None:
        if self.trainer.sanity_checking:
            return

        # logs user requested information to logger
        assert not self._epoch_end_reached
        self.log_metrics(self.metrics[MetricSource.LOG], step=self._eval_log_step)

        # increment the step even if nothing was logged
        self._increment_eval_log_step()

    def _prepare_eval_loop_results(self, metrics: Mapping[str, _METRIC]) -> None:
        if self.trainer.sanity_checking:
            return

        num_dataloaders = self.trainer._evaluation_loop.num_dataloaders
        has_been_initialized = len(self.eval_loop_results) == num_dataloaders
        for dl_idx in range(self.trainer._evaluation_loop.num_dataloaders):
            # remove callback metrics that don't belong to this dataloader
            callback_metrics = {
                k: v
                for k, v in metrics.items() if "dataloader_idx" not in k or f"dataloader_idx_{dl_idx}" in k
            }
            if has_been_initialized:
                self.eval_loop_results[dl_idx].update(callback_metrics)
            else:
                self.eval_loop_results.append(callback_metrics)

    def update_eval_epoch_metrics(self) -> _EVALUATE_OUTPUT:
        assert self._epoch_end_reached
        metrics = self.metrics

        if not self.trainer.sanity_checking:
            # log all the metrics as a single dict
            self.log_metrics(metrics[MetricSource.LOG])

        self._prepare_eval_loop_results(metrics[MetricSource.CALLBACK])

        # log results of evaluation
        if (
            self.trainer.state.fn != TrainerFn.FITTING and self.trainer.evaluating and self.trainer.is_global_zero
            and self.trainer.verbose_evaluate
        ):
            print('-' * 80)
            for result_idx, results in enumerate(self.eval_loop_results):
                print(f'DATALOADER:{result_idx} {self.trainer.state.stage.upper()} RESULTS')
                pprint({
                    k: (v.item() if v.numel() == 1 else v.tolist()) if isinstance(v, torch.Tensor) else v
                    for k, v in results.items()
                })
                print('-' * 80)

        results = self.eval_loop_results
        # clear mem
        self.eval_loop_results = []
        return results

    """
    Train metric updates
    """

    def on_train_split_start(self, batch_idx: int, split_idx: int, split_batch: Any) -> None:
        self.trainer._results.extract_batch_size(split_batch)
        self._batch_idx = batch_idx
        self._split_idx = split_idx

    def update_train_step_metrics(self) -> None:
        if self.trainer.fit_loop.should_accumulate() and self.trainer.lightning_module.automatic_optimization:
            return

        self._log_gpus_metrics()

        # when metrics should be logged
        assert not self._epoch_end_reached
        if self.should_update_logs or self.trainer.fast_dev_run:
            self.log_metrics(self.metrics[MetricSource.LOG])

    def update_train_epoch_metrics(self) -> None:
        # add the metrics to the loggers
        assert self._epoch_end_reached
        self.log_metrics(self.metrics[MetricSource.LOG])

        # reset result collection for next epoch
        self.trainer._results.reset(metrics=True)

    def _log_gpus_metrics(self):
        for key, mem in self.gpus_metrics.items():
            gpu_id = int(key.split('/')[0].split(':')[1])
            if gpu_id in self.trainer.accelerator_connector.parallel_device_ids:
                self.trainer.lightning_module.log(key, mem, prog_bar=False, logger=True, on_step=True, on_epoch=False)

    """
    Utilities and properties
    """

    def on_epoch_start(self) -> None:
        self._epoch_end_reached = False

    def on_batch_start(self) -> None:
        self._epoch_end_reached = False

    def epoch_end_reached(self):
        self.trainer.logger_connector._epoch_end_reached = True
        self.trainer.logger_connector._batch_idx = None
        self.trainer.logger_connector._split_idx = None

    def on_epoch_end(self) -> None:
        assert self._epoch_end_reached
        metrics = self.metrics
        self._progress_bar_metrics.update(metrics[MetricSource.PBAR])
        self._callback_metrics.update(metrics[MetricSource.CALLBACK])
        self._logged_metrics.update(metrics[MetricSource.LOG])
        self._current_fx = None

    def on_batch_end(self) -> None:
        assert not self._epoch_end_reached
        metrics = self.metrics
        self._progress_bar_metrics.update(metrics[MetricSource.PBAR])
        self._callback_metrics.update(metrics[MetricSource.CALLBACK])
        self._logged_metrics.update(metrics[MetricSource.LOG])

    def should_reset_tensors(self, fx: str) -> bool:
        is_different_fx = self._current_fx != fx
        if self._split_idx is None:
            is_first_batch = self._batch_idx in (None, 0)
        else:
            is_first_batch = self._batch_idx + self._split_idx == 0
        return is_different_fx and is_first_batch

    def reset(self, metrics: Optional[bool] = None) -> None:
        if self.trainer.sanity_checking:
            # reset metrics
            self._progress_bar_metrics = {}
            self._logged_metrics = {}
            self._callback_metrics = {}
        self.trainer._results.reset(metrics=metrics)
        self._batch_idx = None
        self._split_idx = None
        self._current_fx = None

    @property
    def metrics(self) -> Dict[MetricSource, Dict[str, _METRIC]]:
        """This function returns either batch or epoch metrics depending on ``_epoch_end_reached``."""
        on_step = not self._epoch_end_reached
        return self.trainer._results.metrics(on_step)

    @property
    def gpus_metrics(self) -> Dict[str, str]:
        if self.trainer._device_type == DeviceType.GPU and self.log_gpu_memory:
            mem_map = memory.get_memory_profile(self.log_gpu_memory)
            self._gpus_metrics.update(mem_map)
        return self._gpus_metrics

    @property
    def callback_metrics(self) -> Dict[str, _METRIC]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.CALLBACK]
            self._callback_metrics.update(metrics)
        return self._callback_metrics

    @property
    def logged_metrics(self) -> Dict[str, _METRIC]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.LOG]
            self._logged_metrics.update(metrics)
        return self._logged_metrics

    @property
    def progress_bar_metrics(self) -> Dict[str, float]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.PBAR]
            self._progress_bar_metrics.update(metrics)
        return self._progress_bar_metrics
<commit_msg>fix logger type hint (#8359)<commit_after># Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from pprint import pprint
from typing import Any, Dict, Iterable, Mapping, Optional, Union

import torch

import pytorch_lightning as pl
from pytorch_lightning.core import memory
from pytorch_lightning.loggers import LightningLoggerBase, LoggerCollection, TensorBoardLogger
from pytorch_lightning.trainer.connectors.logger_connector.result import _METRIC, MetricSource
from pytorch_lightning.trainer.states import RunningStage, TrainerFn
from pytorch_lightning.utilities import DeviceType
from pytorch_lightning.utilities.metrics import metrics_to_scalars
from pytorch_lightning.utilities.types import _EVALUATE_OUTPUT

class LoggerConnector:

    def __init__(self, trainer: 'pl.Trainer', log_gpu_memory: Optional[str] = None) -> None:
        self.trainer = trainer
        self.log_gpu_memory = log_gpu_memory
        self.eval_loop_results = []
        self._val_log_step: int = 0
        self._test_log_step: int = 0
        self._progress_bar_metrics: Dict[str, float] = {}
        self._logged_metrics: Dict[str, _METRIC] = {}
        self._callback_metrics: Dict[str, _METRIC] = {}
        self._gpus_metrics: Dict[str, str] = {}
        self._epoch_end_reached = False
        self._current_fx: Optional[str] = None
        self._batch_idx: Optional[int] = None
        self._split_idx: Optional[int] = None

    def on_trainer_init(
        self,
        logger: Union[bool, LightningLoggerBase, Iterable[LightningLoggerBase]],
        flush_logs_every_n_steps: int,
        log_every_n_steps: int,
        move_metrics_to_cpu: bool,
    ) -> None:
        self.configure_logger(logger)
        self.trainer.flush_logs_every_n_steps = flush_logs_every_n_steps
        self.trainer.log_every_n_steps = log_every_n_steps
        self.trainer.move_metrics_to_cpu = move_metrics_to_cpu

    @property
    def should_flush_logs(self) -> bool:
        should_flush = (self.trainer.global_step + 1) % self.trainer.flush_logs_every_n_steps == 0
        return should_flush or self.trainer.should_stop

    @property
    def should_update_logs(self) -> bool:
        should_log_every_n_steps = (self.trainer.global_step + 1) % self.trainer.log_every_n_steps == 0
        return should_log_every_n_steps or self.trainer.should_stop

    def configure_logger(self, logger: Union[bool, LightningLoggerBase, Iterable[LightningLoggerBase]]) -> None:
        if logger is True:
            version = os.environ.get('PL_EXP_VERSION', self.trainer.slurm_job_id)

            # default logger
            self.trainer.logger = TensorBoardLogger(
                save_dir=self.trainer.default_root_dir, version=version, name='lightning_logs'
            )
        elif logger is False:
            self.trainer.logger = None
        else:
            if isinstance(logger, Iterable):
                self.trainer.logger = LoggerCollection(logger)
            else:
                self.trainer.logger = logger

    def log_metrics(self, metrics: Dict[str, _METRIC], step: Optional[int] = None) -> None:
        """Logs the metric dict passed in.
        If `step` parameter is None and `step` key is presented is metrics,
        uses metrics["step"] as a step

        Args:
            metrics: Metric values
            step: Step for which metrics should be logged. Default value is `self.global_step` during training or
                the total validation / test log step count during validation and testing.
        """
        if self.trainer.logger is None or not metrics:
            return

        # turn all tensors to scalars
        scalar_metrics = metrics_to_scalars(metrics)

        if step is None:
            step = scalar_metrics.pop("step", None)
        if step is None:
            # added metrics for convenience
            scalar_metrics.setdefault("epoch", self.trainer.current_epoch)
            step = self.trainer.global_step

        # log actual metrics
        if self.trainer.is_global_zero:
            self.trainer.logger.agg_and_log_metrics(scalar_metrics, step=step)
            self.trainer.logger.save()

        self._logged_metrics.update(scalar_metrics)

    """
    Evaluation metric updates
    """

    @property
    def _eval_log_step(self) -> Optional[int]:
        if self.trainer.state.stage is RunningStage.VALIDATING:
            return self._val_log_step
        if self.trainer.state.stage is RunningStage.TESTING:
            return self._test_log_step
        return None

    def _increment_eval_log_step(self) -> None:
        if self.trainer.state.stage is RunningStage.VALIDATING:
            self._val_log_step += 1
        elif self.trainer.state.stage is RunningStage.TESTING:
            self._test_log_step += 1

    def on_evaluation_batch_start(self, batch: Any, batch_idx: int, dataloader_idx: int, num_dataloaders: int) -> None:
        model = self.trainer.lightning_module
        # set dataloader_idx only if multiple ones
        model._current_dataloader_idx = dataloader_idx if num_dataloaders > 1 else None

        # track batch_size
        self.trainer._results.extract_batch_size(batch)
        self._batch_idx = batch_idx

    def update_eval_step_metrics(self) -> None:
        if self.trainer.sanity_checking:
            return

        # logs user requested information to logger
        assert not self._epoch_end_reached
        self.log_metrics(self.metrics[MetricSource.LOG], step=self._eval_log_step)

        # increment the step even if nothing was logged
        self._increment_eval_log_step()

    def _prepare_eval_loop_results(self, metrics: Mapping[str, _METRIC]) -> None:
        if self.trainer.sanity_checking:
            return

        num_dataloaders = self.trainer._evaluation_loop.num_dataloaders
        has_been_initialized = len(self.eval_loop_results) == num_dataloaders
        for dl_idx in range(self.trainer._evaluation_loop.num_dataloaders):
            # remove callback metrics that don't belong to this dataloader
            callback_metrics = {
                k: v
                for k, v in metrics.items() if "dataloader_idx" not in k or f"dataloader_idx_{dl_idx}" in k
            }
            if has_been_initialized:
                self.eval_loop_results[dl_idx].update(callback_metrics)
            else:
                self.eval_loop_results.append(callback_metrics)

    def update_eval_epoch_metrics(self) -> _EVALUATE_OUTPUT:
        assert self._epoch_end_reached
        metrics = self.metrics

        if not self.trainer.sanity_checking:
            # log all the metrics as a single dict
            self.log_metrics(metrics[MetricSource.LOG])

        self._prepare_eval_loop_results(metrics[MetricSource.CALLBACK])

        # log results of evaluation
        if (
            self.trainer.state.fn != TrainerFn.FITTING and self.trainer.evaluating and self.trainer.is_global_zero
            and self.trainer.verbose_evaluate
        ):
            print('-' * 80)
            for result_idx, results in enumerate(self.eval_loop_results):
                print(f'DATALOADER:{result_idx} {self.trainer.state.stage.upper()} RESULTS')
                pprint({
                    k: (v.item() if v.numel() == 1 else v.tolist()) if isinstance(v, torch.Tensor) else v
                    for k, v in results.items()
                })
                print('-' * 80)

        results = self.eval_loop_results
        # clear mem
        self.eval_loop_results = []
        return results

    """
    Train metric updates
    """

    def on_train_split_start(self, batch_idx: int, split_idx: int, split_batch: Any) -> None:
        self.trainer._results.extract_batch_size(split_batch)
        self._batch_idx = batch_idx
        self._split_idx = split_idx

    def update_train_step_metrics(self) -> None:
        if self.trainer.fit_loop.should_accumulate() and self.trainer.lightning_module.automatic_optimization:
            return

        self._log_gpus_metrics()

        # when metrics should be logged
        assert not self._epoch_end_reached
        if self.should_update_logs or self.trainer.fast_dev_run:
            self.log_metrics(self.metrics[MetricSource.LOG])

    def update_train_epoch_metrics(self) -> None:
        # add the metrics to the loggers
        assert self._epoch_end_reached
        self.log_metrics(self.metrics[MetricSource.LOG])

        # reset result collection for next epoch
        self.trainer._results.reset(metrics=True)

    def _log_gpus_metrics(self):
        for key, mem in self.gpus_metrics.items():
            gpu_id = int(key.split('/')[0].split(':')[1])
            if gpu_id in self.trainer.accelerator_connector.parallel_device_ids:
                self.trainer.lightning_module.log(key, mem, prog_bar=False, logger=True, on_step=True, on_epoch=False)

    """
    Utilities and properties
    """

    def on_epoch_start(self) -> None:
        self._epoch_end_reached = False

    def on_batch_start(self) -> None:
        self._epoch_end_reached = False

    def epoch_end_reached(self):
        self.trainer.logger_connector._epoch_end_reached = True
        self.trainer.logger_connector._batch_idx = None
        self.trainer.logger_connector._split_idx = None

    def on_epoch_end(self) -> None:
        assert self._epoch_end_reached
        metrics = self.metrics
        self._progress_bar_metrics.update(metrics[MetricSource.PBAR])
        self._callback_metrics.update(metrics[MetricSource.CALLBACK])
        self._logged_metrics.update(metrics[MetricSource.LOG])
        self._current_fx = None

    def on_batch_end(self) -> None:
        assert not self._epoch_end_reached
        metrics = self.metrics
        self._progress_bar_metrics.update(metrics[MetricSource.PBAR])
        self._callback_metrics.update(metrics[MetricSource.CALLBACK])
        self._logged_metrics.update(metrics[MetricSource.LOG])

    def should_reset_tensors(self, fx: str) -> bool:
        is_different_fx = self._current_fx != fx
        if self._split_idx is None:
            is_first_batch = self._batch_idx in (None, 0)
        else:
            is_first_batch = self._batch_idx + self._split_idx == 0
        return is_different_fx and is_first_batch

    def reset(self, metrics: Optional[bool] = None) -> None:
        if self.trainer.sanity_checking:
            # reset metrics
            self._progress_bar_metrics = {}
            self._logged_metrics = {}
            self._callback_metrics = {}
        self.trainer._results.reset(metrics=metrics)
        self._batch_idx = None
        self._split_idx = None
        self._current_fx = None

    @property
    def metrics(self) -> Dict[MetricSource, Dict[str, _METRIC]]:
        """This function returns either batch or epoch metrics depending on ``_epoch_end_reached``."""
        on_step = not self._epoch_end_reached
        return self.trainer._results.metrics(on_step)

    @property
    def gpus_metrics(self) -> Dict[str, str]:
        if self.trainer._device_type == DeviceType.GPU and self.log_gpu_memory:
            mem_map = memory.get_memory_profile(self.log_gpu_memory)
            self._gpus_metrics.update(mem_map)
        return self._gpus_metrics

    @property
    def callback_metrics(self) -> Dict[str, _METRIC]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.CALLBACK]
            self._callback_metrics.update(metrics)
        return self._callback_metrics

    @property
    def logged_metrics(self) -> Dict[str, _METRIC]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.LOG]
            self._logged_metrics.update(metrics)
        return self._logged_metrics

    @property
    def progress_bar_metrics(self) -> Dict[str, float]:
        if self.trainer._results:
            metrics = self.metrics[MetricSource.PBAR]
            self._progress_bar_metrics.update(metrics)
        return self._progress_bar_metrics
<|endoftext|>


<commit_before># coding=utf-8
from django.test import TransactionTestCase
import datetime
from hello.models import Message
from hello.reddit import (
    get_companies,
    ticker_to_name,
    scrape_reddit,
    save_reddit_articles
)
import json

class RedditScraper(TransactionTestCase):

    def test_json_loads(self):
        companies = get_companies()
        expected = {
            "Ticker": "MSFT",
            "Name": "Microsoft Corporation",
            "Exchange": "NMS",
            "Country": "USA",
            "Category Name": "Business Software & Services",
            "Category Number": 826,
            "": 0
        }
        self.assertEqual(companies['MSFT'], expected)

    def test_ticker_to_name_works(self):
        companies = get_companies()
        expected = "Microsoft Corporation"
        self.assertEqual(ticker_to_name(companies, "MSFT"), expected)

    def test_ticker_to_name_bad_company_data(self):
        with self.assertRaises(ValueError):
            ticker_to_name("waffles", "MSFT")

    def test_ticker_to_name_bad_ticker(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, "ayyooo")

    def test_reddit_scraper(self):
        links = scrape_reddit("MSFT", "Microsoft Corporation")
        expected = 'https://www.reddit.com/r/linux/comments/mgtht/given_the_recent_protests_shouldnt_we_point_out/'
        self.assertIn(expected, str(links))

    def test_reddit_save(self):
        links = scrape_reddit("AAPL", str(ticker_to_name(get_companies(), "SUNE")))
        expected = {
            'url': 'http://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/?ref=search_posts',
            'urls': ['https://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/'],
            'social_id': '40wx7b',
            'created_time': datetime.datetime.utcfromtimestamp(1452764067.0).strftime("%Y-%m-%dT%H:%M:%SZ"),
            'content': 'Sunedison Inc To Distribute Tesla Motors Inc Powerwall',
            'author': 'MartEden',
            'popularity': 27,
            'source': 'reddit',
            'author_image': 'https://www.redditstatic.com/icon-touch.png',
            'focus': 'AAPL',
            'symbols': ['AAPL']
        }
        save_reddit_articles(links)
        dbobj = Message.objects.get(social_id=expected['social_id'])
        self.assertEqual(dbobj.url, str(expected['url']))
        self.assertEqual(dbobj.content, str(expected['content']))
        self.assertEqual(dbobj.author, str(expected['author']))
        self.assertEqual(dbobj.focus, 'AAPL')

    def test_invalid_ticker_type(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, ("ticker", ))

    def test_duplicate_companies(self):
        links = scrape_reddit("AAPL", str(ticker_to_name(get_companies(), "SUNE")))
        save_reddit_articles(links)
        instance1 = Message.objects.all()
        save_reddit_articles(links)
        instance2 = Message.objects.all()
        self.assertEqual(instance1[0], instance2[0])
        self.assertEqual(instance1[1], instance2[1])

    def test_invalid_ticker_reddit_scrape(self):
        with self.assertRaises(TypeError):
            scrape_reddit(("ticker", ), "query")

    def test_invalid_query_reddit_scrape(self):
        with self.assertRaises(TypeError):
            scrape_reddit("ticker", ("query", ))
<commit_msg>Changed expected link in reddit<commit_after># coding=utf-8
from django.test import TransactionTestCase
import datetime
from hello.models import Message
from hello.reddit import (
    get_companies,
    ticker_to_name,
    scrape_reddit,
    save_reddit_articles
)
import json

class RedditScraper(TransactionTestCase):

    def test_json_loads(self):
        companies = get_companies()
        expected = {
            "Ticker": "MSFT",
            "Name": "Microsoft Corporation",
            "Exchange": "NMS",
            "Country": "USA",
            "Category Name": "Business Software & Services",
            "Category Number": 826,
            "": 0
        }
        self.assertEqual(companies['MSFT'], expected)

    def test_ticker_to_name_works(self):
        companies = get_companies()
        expected = "Microsoft Corporation"
        self.assertEqual(ticker_to_name(companies, "MSFT"), expected)

    def test_ticker_to_name_bad_company_data(self):
        with self.assertRaises(ValueError):
            ticker_to_name("waffles", "MSFT")

    def test_ticker_to_name_bad_ticker(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, "ayyooo")

    def test_reddit_scraper(self):
        links = scrape_reddit("MSFT", "Microsoft Corporation")
        expected = 'https://www.reddit.com/r/investing/comments/47e1x2/you_can_buy_uber_airbnb_pinterest_right_now/'
        self.assertIn(expected, str(links))

    def test_reddit_save(self):
        links = scrape_reddit("AAPL", str(ticker_to_name(get_companies(), "SUNE")))
        expected = {
            'url': 'http://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/?ref=search_posts',
            'urls': ['https://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/'],
            'social_id': '40wx7b',
            'created_time': datetime.datetime.utcfromtimestamp(1452764067.0).strftime("%Y-%m-%dT%H:%M:%SZ"),
            'content': 'Sunedison Inc To Distribute Tesla Motors Inc Powerwall',
            'author': 'MartEden',
            'popularity': 27,
            'source': 'reddit',
            'author_image': 'https://www.redditstatic.com/icon-touch.png',
            'focus': 'AAPL',
            'symbols': ['AAPL']
        }
        save_reddit_articles(links)
        dbobj = Message.objects.get(social_id=expected['social_id'])
        self.assertEqual(dbobj.url, str(expected['url']))
        self.assertEqual(dbobj.content, str(expected['content']))
        self.assertEqual(dbobj.author, str(expected['author']))
        self.assertEqual(dbobj.focus, 'AAPL')

    def test_invalid_ticker_type(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, ("ticker", ))

    def test_duplicate_companies(self):
        links = scrape_reddit("AAPL", str(ticker_to_name(get_companies(), "SUNE")))
        save_reddit_articles(links)
        instance1 = Message.objects.all()
        save_reddit_articles(links)
        instance2 = Message.objects.all()
        self.assertEqual(instance1[0], instance2[0])
        self.assertEqual(instance1[1], instance2[1])

    def test_invalid_ticker_reddit_scrape(self):
        with self.assertRaises(TypeError):
            scrape_reddit(("ticker", ), "query")

    def test_invalid_query_reddit_scrape(self):
        with self.assertRaises(TypeError):
            scrape_reddit("ticker", ("query", ))
<|endoftext|>