<commit_before>
import datetime

from django.shortcuts import render
from django.template.loader import render_to_string

from viewcore import viewcore

def get_monats_namen(monat):
    return datetime.date(1900, monat, 1).strftime('%B')

def _umrechnen(tabelle):
    result = {}
    for tblindex, group in tabelle.iterrows():
        print(index)
        print('###')
        print(group)
        (datum, kategorie) = tblindex
        if datum not in result:
            result[datum] = {}
        result[datum][kategorie] = group.Wert
    return result

def _computePieChartProzentual(context, jahr):
    result = viewcore.database_instance().einzelbuchungen.get_jahresausgaben_nach_kategorie_prozentual(jahr)
    ausgaben_data = []
    ausgaben_labels = []
    ausgaben_colors = []
    for kategorie, wert in result.items():
        ausgaben_data.append('%.2f' % abs(wert))
        ausgaben_labels.append(kategorie)
        ausgaben_colors.append('#' + viewcore.database_instance().einzelbuchungen.get_farbe_fuer(kategorie))

    context['pie_ausgaben_data_prozentual'] = ausgaben_data
    context['pie_ausgaben_labels'] = ausgaben_labels
    context['pie_ausgaben_colors'] = ausgaben_colors

    result = viewcore.database_instance().einzelbuchungen.get_jahreseinnahmen_nach_kategorie_prozentual(jahr)
    einnahmen_data = []
    einnahmen_labels = []
    einnahmen_colors = []
    for kategorie, wert in result.items():
        einnahmen_data.append('%.2f' % abs(wert))
        einnahmen_labels.append(kategorie)
        einnahmen_colors.append('#' + viewcore.database_instance().einzelbuchungen.get_farbe_fuer(kategorie))

    context['pie_einnahmen_data_prozentual'] = einnahmen_data
    context['pie_einnahmen_labels'] = einnahmen_labels
    context['pie_einnahmen_colors'] = einnahmen_colors

    return context

def _compile_colors(result, einzelbuchungen):
    einnahmen = {}
    for month in result.keys():
        for kategorie in result[month]:
            if month == 1:
                einnahmen[kategorie] = {}
                einnahmen[kategorie]['values'] = '[' + result[month][kategorie]
                continue
            einnahmen[kategorie]['values'] = einnahmen[kategorie]['values'] + ',' + result[month][kategorie]

    for kategorie in einnahmen:
        einnahmen[kategorie]['values'] = einnahmen[kategorie]['values'] + ']'
        einnahmen[kategorie]['farbe'] = einzelbuchungen.get_farbe_fuer(kategorie)

    return einnahmen

def index(request):
    context = handle_request(request)
    print(context)
    rendered_content = render_to_string('theme/uebersicht_jahr.html', context, request=request)

    context['content'] = rendered_content

    return render(request, 'theme/index.html', context)

def handle_request(request):
    today = datetime.date.today()
    year = today.year

    if 'date' in request.POST:
        year = int(float(request.POST['date']))
    einzelbuchungen = viewcore.database_instance().einzelbuchungen

    jahresbuchungs_tabelle = einzelbuchungen.select().select_year(year)
    jahres_ausgaben = jahresbuchungs_tabelle.select_ausgaben()
    jahres_einnahmen = jahresbuchungs_tabelle.select_einnahmen()

    jahresausgaben = []
    for kategorie, jahresblock in jahres_ausgaben.group_by_kategorie().iterrows():
        jahresausgaben.append([kategorie, '%.2f' % jahresblock.Wert, einzelbuchungen.get_farbe_fuer(kategorie)])

    jahreseinnahmen = []
    for kategorie, jahresblock in jahres_einnahmen.group_by_kategorie().iterrows():
        jahreseinnahmen.append([kategorie, '%.2f' % jahresblock.Wert, einzelbuchungen.get_farbe_fuer(kategorie)])

    num_monate = sorted(list(set(jahresbuchungs_tabelle.raw_table().Datum.map(lambda x: x.month))))
    monats_namen = []
    for num_monat in num_monate:
        monats_namen.append(get_monats_namen(num_monat))

    context = viewcore.generate_base_context('jahresuebersicht')

    context['durchschnitt_monat_kategorien'] = str(list(einzelbuchungen.durchschnittliche_ausgaben_pro_monat(year).keys()))
    context['durchschnittlich_monat_wert'] = str(list(einzelbuchungen.durchschnittliche_ausgaben_pro_monat(year).values()))
    context = _computePieChartProzentual(context, year)

    laenge = 12
    if year == today.year:
        laenge = today.month

    context['buchungen'] = [
        {
            'kategorie': 'Einnahmen',
            'farbe': 'rgb(210, 214, 222)',
            'wert': jahres_einnahmen.inject_zeros_for_year(year, laenge).sum_monthly()
        },
        {
            'kategorie': 'Ausgaben',
            'farbe': 'rgba(60,141,188,0.8)',
            'wert': jahres_ausgaben.inject_zeros_for_year(year, laenge).sum_monthly()
        }
    ]
    context['zusammenfassung_ausgaben'] = jahresausgaben
    context['zusammenfassung_einnahmen'] = jahreseinnahmen
    context['monats_namen'] = monats_namen
    context['selected_date'] = year

    context['einnahmen'] = _compile_colors(jahres_einnahmen.inject_zeroes_for_year_and_kategories(2017).sum_kategorien_monthly(), einzelbuchungen)
    context['ausgaben'] = _compile_colors(jahres_ausgaben.inject_zeroes_for_year_and_kategories(2017).sum_kategorien_monthly(), einzelbuchungen)
    context['jahre'] = sorted(einzelbuchungen.get_jahre(), reverse=True)
    context['gesamt_ausgaben'] = '%.2f' % einzelbuchungen.select().select_year(year).select_ausgaben().sum()
    context['gesamt_einnahmen'] = '%.2f' % einzelbuchungen.select().select_year(year).select_einnahmen().sum()
    return context
<commit_msg>removed outdated function<commit_after>
import datetime

from django.shortcuts import render
from django.template.loader import render_to_string

from viewcore import viewcore

def get_monats_namen(monat):
    return datetime.date(1900, monat, 1).strftime('%B')

def _computePieChartProzentual(context, jahr):
    result = viewcore.database_instance().einzelbuchungen.get_jahresausgaben_nach_kategorie_prozentual(jahr)
    ausgaben_data = []
    ausgaben_labels = []
    ausgaben_colors = []
    for kategorie, wert in result.items():
        ausgaben_data.append('%.2f' % abs(wert))
        ausgaben_labels.append(kategorie)
        ausgaben_colors.append('#' + viewcore.database_instance().einzelbuchungen.get_farbe_fuer(kategorie))

    context['pie_ausgaben_data_prozentual'] = ausgaben_data
    context['pie_ausgaben_labels'] = ausgaben_labels
    context['pie_ausgaben_colors'] = ausgaben_colors

    result = viewcore.database_instance().einzelbuchungen.get_jahreseinnahmen_nach_kategorie_prozentual(jahr)
    einnahmen_data = []
    einnahmen_labels = []
    einnahmen_colors = []
    for kategorie, wert in result.items():
        einnahmen_data.append('%.2f' % abs(wert))
        einnahmen_labels.append(kategorie)
        einnahmen_colors.append('#' + viewcore.database_instance().einzelbuchungen.get_farbe_fuer(kategorie))

    context['pie_einnahmen_data_prozentual'] = einnahmen_data
    context['pie_einnahmen_labels'] = einnahmen_labels
    context['pie_einnahmen_colors'] = einnahmen_colors

    return context

def _compile_colors(result, einzelbuchungen):
    einnahmen = {}
    for month in result.keys():
        for kategorie in result[month]:
            if month == 1:
                einnahmen[kategorie] = {}
                einnahmen[kategorie]['values'] = '[' + result[month][kategorie]
                continue
            einnahmen[kategorie]['values'] = einnahmen[kategorie]['values'] + ',' + result[month][kategorie]

    for kategorie in einnahmen:
        einnahmen[kategorie]['values'] = einnahmen[kategorie]['values'] + ']'
        einnahmen[kategorie]['farbe'] = einzelbuchungen.get_farbe_fuer(kategorie)

    return einnahmen

def index(request):
    context = handle_request(request)
    print(context)
    rendered_content = render_to_string('theme/uebersicht_jahr.html', context, request=request)

    context['content'] = rendered_content

    return render(request, 'theme/index.html', context)

def handle_request(request):
    today = datetime.date.today()
    year = today.year

    if 'date' in request.POST:
        year = int(float(request.POST['date']))
    einzelbuchungen = viewcore.database_instance().einzelbuchungen

    jahresbuchungs_tabelle = einzelbuchungen.select().select_year(year)
    jahres_ausgaben = jahresbuchungs_tabelle.select_ausgaben()
    jahres_einnahmen = jahresbuchungs_tabelle.select_einnahmen()

    jahresausgaben = []
    for kategorie, jahresblock in jahres_ausgaben.group_by_kategorie().iterrows():
        jahresausgaben.append([kategorie, '%.2f' % jahresblock.Wert, einzelbuchungen.get_farbe_fuer(kategorie)])

    jahreseinnahmen = []
    for kategorie, jahresblock in jahres_einnahmen.group_by_kategorie().iterrows():
        jahreseinnahmen.append([kategorie, '%.2f' % jahresblock.Wert, einzelbuchungen.get_farbe_fuer(kategorie)])

    num_monate = sorted(list(set(jahresbuchungs_tabelle.raw_table().Datum.map(lambda x: x.month))))
    monats_namen = []
    for num_monat in num_monate:
        monats_namen.append(get_monats_namen(num_monat))

    context = viewcore.generate_base_context('jahresuebersicht')

    context['durchschnitt_monat_kategorien'] = str(list(einzelbuchungen.durchschnittliche_ausgaben_pro_monat(year).keys()))
    context['durchschnittlich_monat_wert'] = str(list(einzelbuchungen.durchschnittliche_ausgaben_pro_monat(year).values()))
    context = _computePieChartProzentual(context, year)

    laenge = 12
    if year == today.year:
        laenge = today.month

    context['buchungen'] = [
        {
            'kategorie': 'Einnahmen',
            'farbe': 'rgb(210, 214, 222)',
            'wert': jahres_einnahmen.inject_zeros_for_year(year, laenge).sum_monthly()
        },
        {
            'kategorie': 'Ausgaben',
            'farbe': 'rgba(60,141,188,0.8)',
            'wert': jahres_ausgaben.inject_zeros_for_year(year, laenge).sum_monthly()
        }
    ]
    context['zusammenfassung_ausgaben'] = jahresausgaben
    context['zusammenfassung_einnahmen'] = jahreseinnahmen
    context['monats_namen'] = monats_namen
    context['selected_date'] = year

    context['einnahmen'] = _compile_colors(jahres_einnahmen.inject_zeroes_for_year_and_kategories(2017).sum_kategorien_monthly(), einzelbuchungen)
    context['ausgaben'] = _compile_colors(jahres_ausgaben.inject_zeroes_for_year_and_kategories(2017).sum_kategorien_monthly(), einzelbuchungen)
    context['jahre'] = sorted(einzelbuchungen.get_jahre(), reverse=True)
    context['gesamt_ausgaben'] = '%.2f' % einzelbuchungen.select().select_year(year).select_ausgaben().sum()
    context['gesamt_einnahmen'] = '%.2f' % einzelbuchungen.select().select_year(year).select_einnahmen().sum()
    return context
<|endoftext|>


<commit_before>"""Helper functions for retrieving data via the GitHub API.

auth() -----------> Return credentials for use in GitHub API calls.
auth_user() ------> Set GitHub user for subsequent API calls.

memberfields() ---> Get field values for a member/user.
members() --------> Get members of one or more organizations.
membersget() -----> Get member info for a specified organization.

pagination() -----> Parse values from 'link' HTTP header returned by GitHub API.

repofields() -----> Get field values for a repo.
repos() ----------> Get repo information for organization(s) or user(s).
reposget() -------> Get repo information from specified API endpoint.

verbose() --------> Set verbose mode on/off.
verbose_output() -> Display status info in verbose mode.

write_csv() ------> Write a list of namedtuples to a CSV file.
"""
import collections
import csv
import json
import traceback

import requests

#------------------------------------------------------------------------------
class _settings: # pylint: disable=R0903
    """Used for global settings. Should not be accessed directly - e.g.,
    use the verbose() function to change _settings.verbose, or the user()
    function to change _settings.github_username/accesstoken.
    """
    verbose = False # default = verbose mode off
    github_username = None # default = no GitHub authentication
    github_accesstoken = None

#------------------------------------------------------------------------------
def auth():
    """Credentials for basic authentication.

    Returns the tuple used for API calls, based on current settings.
    Returns None if no GitHub username/PAT is current set.
    """
    # Note that "auth() ->" is explcitly added to the verbose_output()
    # calls below because auth() is typically used inline from other
    # functions, so it isn't the caller in the call stack.

    if not _settings.github_username:
        return None

    username = _settings.github_username
    access_token = _settings.github_accesstoken

    verbose_output('auth() ->', 'username:', username + ', PAT:',
                   access_token[0:2] + '...' + access_token[-2:])

    return (username, access_token)

#-------------------------------------------------------------------------------
def auth_user(username=None):
    """Set GitHub user for subsequent API calls.

    username = a GitHub username stored in github_users.json in the
               /private subfolder. If omitted or None, resets GitHub user to
               no authentication.
    """
    if username:
        with open('private/github_users.json', 'r') as jsonfile:
            github_users = json.load(jsonfile)
            _settings.github_username = username
            _settings.github_accesstoken = github_users[username]
    else:
        # no username specified, so reset to default of no authentication
        _settings.github_username = None
        _settings.github_accesstoken = None

#-------------------------------------------------------------------------------
def memberfields(member_json, fields, org):
    """Get field values for a member/user.

    1st parameter = member's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields
    3rd parameter = organization ID

    Returns a namedtuple containing the desired fields and their values.
    NOTE: in addition to the specified fields, always returns an 'org' field
    to distinguish between organizations in lists returned by members().
    """
    values = {}
    values['org'] = org
    for fldname in fields:
        values[fldname] = member_json[fldname]

    member_tuple = collections.namedtuple('member_tuple',
                                          'org ' + ' '.join(fields))
    return member_tuple(**values)

#-------------------------------------------------------------------------------
def members(org=None, fields=None, audit2fa=False):
    """Get members for one or more organizations.

    org = organization
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API
    audit2fa = whether to only return members with 2FA disabled. You must be
               authenticated via auth_user() as an admin of the org(s) to use
               this option.

    Returns a list of namedtuple objects, one per member.
    """
    if not fields:
        fields = ['login', 'id', 'type', 'site_admin'] # default field list

    memberlist = [] # the list of members that will be returned

    # org may be a single value as a string, or a list of values
    if isinstance(org, str):
        memberlist.extend(membersget(org, fields, audit2fa))
    else:
        for orgid in org:
            memberlist.extend(membersget(orgid, fields, audit2fa))

    return memberlist

#------------------------------------------------------------------------------
def membersget(org, fields, audit2fa=False):
    """Get member info for a specified organization.

    1st parameter = organization ID
    2nd parameter = list of fields to be returned
    audit2fa = whether to only return members with 2FA disabled.
               Note: for audit2fa=True, you must be authenticated via
               auth_user() as an admin of the org(s).

    Returns a list of namedtuples containing the specified fields.
    """
    endpoint = 'https://api.github.com/orgs/' + org + '/members' + \
        ('?filter=2fa_disabled' if audit2fa else '')
    retval = [] # the list to be returned
    totpages = 0

    while True:

        # GitHub API call
        response = requests.get(endpoint, auth=auth())
        verbose_output('API rate limit: {0}, remaining: {1}'. \
            format(response.headers['X-RateLimit-Limit'],
                   response.headers['X-RateLimit-Remaining']))

        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for member_json in thispage:
                retval.append(memberfields(member_json, fields, org))

        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # no more results to process

        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed: {0}, total members: {1}'. \
        format(totpages, len(retval)))

    return retval

#------------------------------------------------------------------------------
def pagination(link_header):
    """Parse values from the 'link' HTTP header returned by GitHub API.

    1st parameter = either of these options ...
                    - 'link' HTTP header passed as a string
                    - response object returned by requests.get()

    Returns a dictionary with entries for the URLs and page numbers parsed
    from the link string: firstURL, firstpage, prevURL, prevpage, nextURL,
    nextpage, lastURL, lastpage.
    """
    # initialize the dictionary
    retval = {'firstpage':0, 'firstURL':None, 'prevpage':0, 'prevURL':None,
              'nextpage':0, 'nextURL':None, 'lastpage':0, 'lastURL':None}

    if isinstance(link_header, str):
        link_string = link_header
    else:
        # link_header is a response object, get its 'link' HTTP header
        try:
            link_string = link_header.headers['Link']
        except KeyError:
            return retval # no Link HTTP header found, nothing to parse

    links = link_string.split(',')
    for link in links:
        # link format = '<url>; rel="type"'
        linktype = link.split(';')[-1].split('=')[-1].strip()[1:-1]
        url = link.split(';')[0].strip()[1:-1]
        pageno = url.split('?')[-1].split('=')[-1].strip()

        retval[linktype + 'page'] = pageno
        retval[linktype + 'URL'] = url

    return retval

#-------------------------------------------------------------------------------
def repofields(repo_json, fields):
    """Get field values for a repo.

    1st parameter = repo's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields

    Returns a namedtuple containing the desired fields and their values.
    """

    # change '.' to '_' because can't have '.' in an identifier
    fldnames = [_.replace('.', '_') for _ in fields]

    repo_tuple = collections.namedtuple('Repo', ' '.join(fldnames))

    values = {}
    for fldname in fields:
        if '.' in fldname:
            # special case - embedded field within a JSON object
            try:
                values[fldname.replace('.', '_')] = \
                    repo_json[fldname.split('.')[0]][fldname.split('.')[1]]
            except TypeError:
                values[fldname.replace('.', '_')] = None
        else:
            # simple case: copy a value from the JSON to the namedtuple
            values[fldname] = repo_json[fldname]

    return repo_tuple(**values)

#-------------------------------------------------------------------------------
def repos(org=None, user=None, fields=None):
    """Get repo information for organization(s) or user(s).

    org    = organization; an organization or list of organizations
    user   = username; a username or list of usernames
    fields = list of fields to be returned; names must be the same as
             returned by the GitHub API.
             Note: dot notation for embedded elements is supported.
             For example, pass a field named 'license.name' to get the 'name'
             element of the 'license' entry for each repo.

    Returns a list of namedtuple objects, one per repo.
    """
    if not fields:
        fields = ['full_name', 'watchers', 'forks', 'open_issues'] # default

    repolist = [] # the list that will be returned

    if org:
        # get repos by organization
        if isinstance(org, str):
            # one organization
            endpoint = 'https://api.github.com/orgs/' + org + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of organizations
            for orgid in org:
                endpoint = 'https://api.github.com/orgs/' + orgid + '/repos'
                repolist.extend(reposget(endpoint, fields))
    else:
        # get repos by user
        if isinstance(user, str):
            # one user
            endpoint = 'https://api.github.com/users/' + user + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of users
            for userid in user:
                endpoint = 'https://api.github.com/users/' + userid + '/repos'
                repolist.extend(reposget(endpoint, fields))

    return repolist

#-------------------------------------------------------------------------------
def reposget(endpoint, fields):
    """Get repo information from specified API endpoint.

    1st parameter = GitHub API endpoint
    2nd parameter = list of fields to be returned

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    # custom header to retrieve license info while License API is in preview
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}

    while True:

        # GitHub API call
        response = requests.get(endpoint, auth=auth(), headers=headers)
        verbose_output('API rate limit: {0}, remaining: {1}'. \
            format(response.headers['X-RateLimit-Limit'],
                   response.headers['X-RateLimit-Remaining']))

        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for repo_json in thispage:
                retval.append(repofields(repo_json, fields))

        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process

        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed: {0}, total members: {1}'. \
        format(totpages, len(retval)))

    return retval

#-------------------------------------------------------------------------------
def verbose(*args):
    """Set verbose mode on/off.

    1st parameter = True for verbose mode, False to turn verbose mode off.

    Returns the current verbose mode setting as True/False. To query the
    current setting, call verbose() with no parameters.
    """
    if len(args) == 1:
        _settings.verbose = args[0]

    return _settings.verbose

#-------------------------------------------------------------------------------
def verbose_output(*args):
    """Display status information in verbose mode.

    parameters = message to be displayed if verbose(True) is set.

    NOTE: can pass any number of parameters, which will be displayed as a single
    string delimited by spaces.
    """
    if not _settings.verbose:
        return # verbose mode is off

    # convert all arguments to strings. so they can be .join()ed
    string_args = [str(_) for _ in args]

    # get the caller of verbose_output(), which is displayed with the message
    caller = traceback.format_stack()[1].split(',')[2].strip().split()[1]

    print(caller + '() ->', ' '.join(string_args))

#-------------------------------------------------------------------------------
def write_csv(listobj, filename):
    """Write a list of namedtuples to a CSV file.

    1st parameter = the list
    2nd parameter = name of CSV file to be written
    """
    csvfile = open(filename, 'w', newline='')

    csvwriter = csv.writer(csvfile, dialect='excel')
    header_row = listobj[0]._fields
    csvwriter.writerow(header_row)

    for row in listobj:
        values = []
        for fldname in header_row:
            values.append(getattr(row, fldname))
        csvwriter.writerow(values)

    csvfile.close()

    verbose_output('filename:', filename)
    verbose_output('columns:', header_row)
    verbose_output('total rows:', len(listobj))

#===============================================================================
#------------------------------------ TESTS ------------------------------------
#===============================================================================

#-------------------------------------------------------------------------------
def test_auth():
    """Simple test for auth() function.
    """
    print(auth())

#-------------------------------------------------------------------------------
def test_members():
    """Simple test for members() function.
    """
    print(members(org=['bitstadium', 'liveservices']))
    print('total members returned:', len(members(org=['bitstadium', 'liveservices'])))

#-------------------------------------------------------------------------------
def test_repos():
    """Simple test for repos() function.
    """
    oct_repos = repos(user=['octocat', 'dmahugh'],
                      fields=['full_name', 'license.name', 'license'])
    for repo in oct_repos:
        print(repo)

#-------------------------------------------------------------------------------
def test_pagination():
    """Simple test for pagination() function.
    """
    testlinks = "<https://api.github.com/organizations/6154722/" + \
        "repos?page=2>; rel=\"next\", <https://api.github.com/" + \
        "organizations/6154722/repos?page=18>; rel=\"last\""
    print(pagination(testlinks))

# if running standalone, run tests ---------------------------------------------
if __name__ == "__main__":

    verbose(True) # turn on verbose mode
    auth_user('dmahugh')
    #test_auth()
    #test_members()
    test_repos()
    #test_pagination()
<commit_msg>document API field names<commit_after>"""Helper functions for retrieving data via the GitHub API.

auth() -----------> Return credentials for use in GitHub API calls.
auth_user() ------> Set GitHub user for subsequent API calls.

memberfields() ---> Get field values for a member/user.
members() --------> Get members of one or more organizations.
membersget() -----> Get member info for a specified organization.

pagination() -----> Parse values from 'link' HTTP header returned by GitHub API.

repofields() -----> Get field values for a repo.
repos() ----------> Get repo information for organization(s) or user(s).
reposget() -------> Get repo information from specified API endpoint.

verbose() --------> Set verbose mode on/off.
verbose_output() -> Display status info in verbose mode.

write_csv() ------> Write a list of namedtuples to a CSV file.
"""
import collections
import csv
import json
import traceback

import requests

#------------------------------------------------------------------------------
class _settings: # pylint: disable=R0903
    """Used for global settings. Should not be accessed directly - e.g.,
    use the verbose() function to change _settings.verbose, or the user()
    function to change _settings.github_username/accesstoken.
    """
    verbose = False # default = verbose mode off
    github_username = None # default = no GitHub authentication
    github_accesstoken = None

#------------------------------------------------------------------------------
def auth():
    """Credentials for basic authentication.

    Returns the tuple used for API calls, based on current settings.
    Returns None if no GitHub username/PAT is current set.
    """
    # Note that "auth() ->" is explcitly added to the verbose_output()
    # calls below because auth() is typically used inline from other
    # functions, so it isn't the caller in the call stack.

    if not _settings.github_username:
        return None

    username = _settings.github_username
    access_token = _settings.github_accesstoken

    verbose_output('auth() ->', 'username:', username + ', PAT:',
                   access_token[0:2] + '...' + access_token[-2:])

    return (username, access_token)

#-------------------------------------------------------------------------------
def auth_user(username=None):
    """Set GitHub user for subsequent API calls.

    username = a GitHub username stored in github_users.json in the
               /private subfolder. If omitted or None, resets GitHub user to
               no authentication.
    """
    if username:
        with open('private/github_users.json', 'r') as jsonfile:
            github_users = json.load(jsonfile)
            _settings.github_username = username
            _settings.github_accesstoken = github_users[username]
    else:
        # no username specified, so reset to default of no authentication
        _settings.github_username = None
        _settings.github_accesstoken = None

#-------------------------------------------------------------------------------
def memberfields(member_json, fields, org):
    """Get field values for a member/user.

    1st parameter = member's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields
    3rd parameter = organization ID

    Returns a namedtuple containing the desired fields and their values.
    NOTE: in addition to the specified fields, always returns an 'org' field
    to distinguish between organizations in lists returned by members().
    """
    values = {}
    values['org'] = org
    for fldname in fields:
        values[fldname] = member_json[fldname]

    member_tuple = collections.namedtuple('member_tuple',
                                          'org ' + ' '.join(fields))
    return member_tuple(**values)

#-------------------------------------------------------------------------------
def members(org=None, fields=None, audit2fa=False):
    """Get members for one or more organizations.

    org = organization
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API (see below).
    audit2fa = whether to only return members with 2FA disabled. You must be
               authenticated via auth_user() as an admin of the org(s) to use
               this option.

    Returns a list of namedtuple objects, one per member.

    GitHub API fields (as of March 2016):
    id                  events_url          organizations_url
    login               followers_url       received_events_url
    site_admin          following_url       repos_url
    type                gists_url           starred_url
    url                 gravatar_id         subscriptions_url
    avatar_url          html_url
    """
    if not fields:
        fields = ['login', 'id', 'type', 'site_admin'] # default field list

    memberlist = [] # the list of members that will be returned

    # org may be a single value as a string, or a list of values
    if isinstance(org, str):
        memberlist.extend(membersget(org, fields, audit2fa))
    else:
        for orgid in org:
            memberlist.extend(membersget(orgid, fields, audit2fa))

    return memberlist

#------------------------------------------------------------------------------
def membersget(org, fields, audit2fa=False):
    """Get member info for a specified organization.

    1st parameter = organization ID
    2nd parameter = list of fields to be returned
    audit2fa = whether to only return members with 2FA disabled.
               Note: for audit2fa=True, you must be authenticated via
               auth_user() as an admin of the org(s).

    Returns a list of namedtuples containing the specified fields.
    """
    endpoint = 'https://api.github.com/orgs/' + org + '/members' + \
        ('?filter=2fa_disabled' if audit2fa else '')
    retval = [] # the list to be returned
    totpages = 0

    while True:

        # GitHub API call
        response = requests.get(endpoint, auth=auth())
        verbose_output('API rate limit: {0}, remaining: {1}'. \
            format(response.headers['X-RateLimit-Limit'],
                   response.headers['X-RateLimit-Remaining']))

        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for member_json in thispage:
                retval.append(memberfields(member_json, fields, org))

        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # no more results to process

        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed: {0}, total members: {1}'. \
        format(totpages, len(retval)))

    return retval

#------------------------------------------------------------------------------
def pagination(link_header):
    """Parse values from the 'link' HTTP header returned by GitHub API.

    1st parameter = either of these options ...
                    - 'link' HTTP header passed as a string
                    - response object returned by requests.get()

    Returns a dictionary with entries for the URLs and page numbers parsed
    from the link string: firstURL, firstpage, prevURL, prevpage, nextURL,
    nextpage, lastURL, lastpage.
    """
    # initialize the dictionary
    retval = {'firstpage':0, 'firstURL':None, 'prevpage':0, 'prevURL':None,
              'nextpage':0, 'nextURL':None, 'lastpage':0, 'lastURL':None}

    if isinstance(link_header, str):
        link_string = link_header
    else:
        # link_header is a response object, get its 'link' HTTP header
        try:
            link_string = link_header.headers['Link']
        except KeyError:
            return retval # no Link HTTP header found, nothing to parse

    links = link_string.split(',')
    for link in links:
        # link format = '<url>; rel="type"'
        linktype = link.split(';')[-1].split('=')[-1].strip()[1:-1]
        url = link.split(';')[0].strip()[1:-1]
        pageno = url.split('?')[-1].split('=')[-1].strip()

        retval[linktype + 'page'] = pageno
        retval[linktype + 'URL'] = url

    return retval

#-------------------------------------------------------------------------------
def repofields(repo_json, fields):
    """Get field values for a repo.

    1st parameter = repo's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields

    Returns a namedtuple containing the desired fields and their values.
    """

    # change '.' to '_' because can't have '.' in an identifier
    fldnames = [_.replace('.', '_') for _ in fields]

    repo_tuple = collections.namedtuple('Repo', ' '.join(fldnames))

    values = {}
    for fldname in fields:
        if '.' in fldname:
            # special case - embedded field within a JSON object
            try:
                values[fldname.replace('.', '_')] = \
                    repo_json[fldname.split('.')[0]][fldname.split('.')[1]]
            except TypeError:
                values[fldname.replace('.', '_')] = None
        else:
            # simple case: copy a value from the JSON to the namedtuple
            values[fldname] = repo_json[fldname]

    return repo_tuple(**values)

#-------------------------------------------------------------------------------
def repos(org=None, user=None, fields=None):
    """Get repo information for organization(s) or user(s).

    org    = organization; an organization or list of organizations
    user   = username; a username or list of usernames
    fields = list of fields to be returned; names must be the same as
             returned by the GitHub API (see below).
             Note: dot notation for embedded elements is supported.
             For example, pass a field named 'license.name' to get the 'name'
             element of the 'license' entry for each repo.

    Returns a list of namedtuple objects, one per repo.

    GitHub API fields (as of March 2016):
    archive_url         git_tags_url         open_issues
    assignees_url       git_url              open_issues_count
    blobs_url           has_downloads        private
    branches_url        has_issues           pulls_url
    clone_url           has_pages            pushed_at
    collaborators_url   has_wiki             releases_url
    commits_url         homepage             size
    compare_url         hooks_url            ssh_url
    contents_url        html_url             stargazers_count
    contributors_url    id                   stargazers_url
    created_at          issue_comment_url    statuses_url
    default_branch      issue_events_url     subscribers_url
    deployments_url     issues_url           subscription_url
    description         keys_url             svn_url
    downloads_url       labels_url           tags_url
    events_url          language             teams_url
    fork                languages_url        trees_url
    forks               master_branch        updated_at
    forks_count         merges_url           url
    forks_url           milestones_url       watchers
    full_name           mirror_url           watchers_count
    git_commits_url     name
    git_refs_url        notifications_url
    -------------------------------------------------------------
    license.featured              permissions.admin
    license.key                   permissions.pull
    license.name                  permissions.push
    license.url
    -------------------------------------------------------------
    owner.avatar_url              owner.organizations_url
    owner.events_url              owner.received_events_url
    owner.followers_url           owner.repos_url
    owner.following_url           owner.site_admin
    owner.gists_url               owner.starred_url
    owner.gravatar_id             owner.subscriptions_url
    owner.html_url                owner.type
    owner.id                      owner.url
    owner.login
    """
    if not fields:
        fields = ['full_name', 'watchers', 'forks', 'open_issues'] # default

    repolist = [] # the list that will be returned

    if org:
        # get repos by organization
        if isinstance(org, str):
            # one organization
            endpoint = 'https://api.github.com/orgs/' + org + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of organizations
            for orgid in org:
                endpoint = 'https://api.github.com/orgs/' + orgid + '/repos'
                repolist.extend(reposget(endpoint, fields))
    else:
        # get repos by user
        if isinstance(user, str):
            # one user
            endpoint = 'https://api.github.com/users/' + user + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of users
            for userid in user:
                endpoint = 'https://api.github.com/users/' + userid + '/repos'
                repolist.extend(reposget(endpoint, fields))

    return repolist

#-------------------------------------------------------------------------------
def reposget(endpoint, fields):
    """Get repo information from specified API endpoint.

    1st parameter = GitHub API endpoint
    2nd parameter = list of fields to be returned

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    # custom header to retrieve license info while License API is in preview
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}

    while True:

        # GitHub API call
        response = requests.get(endpoint, auth=auth(), headers=headers)
        verbose_output('API rate limit: {0}, remaining: {1}'. \
            format(response.headers['X-RateLimit-Limit'],
                   response.headers['X-RateLimit-Remaining']))

        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for repo_json in thispage:
                retval.append(repofields(repo_json, fields))

        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process

        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed: {0}, total members: {1}'. \
        format(totpages, len(retval)))

    return retval

#-------------------------------------------------------------------------------
def verbose(*args):
    """Set verbose mode on/off.

    1st parameter = True for verbose mode, False to turn verbose mode off.

    Returns the current verbose mode setting as True/False. To query the
    current setting, call verbose() with no parameters.
    """
    if len(args) == 1:
        _settings.verbose = args[0]

    return _settings.verbose

#-------------------------------------------------------------------------------
def verbose_output(*args):
    """Display status information in verbose mode.

    parameters = message to be displayed if verbose(True) is set.

    NOTE: can pass any number of parameters, which will be displayed as a single
    string delimited by spaces.
    """
    if not _settings.verbose:
        return # verbose mode is off

    # convert all arguments to strings. so they can be .join()ed
    string_args = [str(_) for _ in args]

    # get the caller of verbose_output(), which is displayed with the message
    caller = traceback.format_stack()[1].split(',')[2].strip().split()[1]

    print(caller + '() ->', ' '.join(string_args))

#-------------------------------------------------------------------------------
def write_csv(listobj, filename):
    """Write a list of namedtuples to a CSV file.

    1st parameter = the list
    2nd parameter = name of CSV file to be written
    """
    csvfile = open(filename, 'w', newline='')

    csvwriter = csv.writer(csvfile, dialect='excel')
    header_row = listobj[0]._fields
    csvwriter.writerow(header_row)

    for row in listobj:
        values = []
        for fldname in header_row:
            values.append(getattr(row, fldname))
        csvwriter.writerow(values)

    csvfile.close()

    verbose_output('filename:', filename)
    verbose_output('columns:', header_row)
    verbose_output('total rows:', len(listobj))

#===============================================================================
#------------------------------------ TESTS ------------------------------------
#===============================================================================

#-------------------------------------------------------------------------------
def test_auth():
    """Simple test for auth() function.
    """
    print(auth())

#-------------------------------------------------------------------------------
def test_members():
    """Simple test for members() function.
    """
    print(members(org=['bitstadium', 'liveservices']))
    print('total members returned:', len(members(org=['bitstadium', 'liveservices'])))

#-------------------------------------------------------------------------------
def test_repos():
    """Simple test for repos() function.
    """
    oct_repos = repos(user=['octocat'],
                      fields=['full_name', 'license.name', 'license', 'permissions.admin'])
    for repo in oct_repos:
        print(repo)

#-------------------------------------------------------------------------------
def test_pagination():
    """Simple test for pagination() function.
    """
    testlinks = "<https://api.github.com/organizations/6154722/" + \
        "repos?page=2>; rel=\"next\", <https://api.github.com/" + \
        "organizations/6154722/repos?page=18>; rel=\"last\""
    print(pagination(testlinks))

# if running standalone, run tests ---------------------------------------------
if __name__ == "__main__":

    verbose(True) # turn on verbose mode
    auth_user('dmahugh')
    #test_auth()
    #test_members()
    test_repos()
    #test_pagination()
<|endoftext|>


<commit_before>"""
This module contains all actor definitions.

.. autosummary::

    Buffer
    Flow
    Host
    Link
    Packet
    Router

.. moduleauthor:: Lan Hongjian <lanhongjianlr@gmail.com>
.. moduleauthor:: Yamei Ou <oym111@gmail.com>
.. moduleauthor:: Samuel Richerd <dondiego152@gmail.com>
.. moduleauthor:: Jan Van Bruggen <jancvanbruggen@gmail.com>
.. moduleauthor:: Junlin Zhang <neicullyn@gmail.com>
"""

from tla_stop_and_wait import StopAndWait
from cs143sim.constants import PACKET_SIZE

class Buffer:
    """Representation of a data storage container

    Buffers store data to be linked while :class:`.Link` is busy sending data.

    :param int capacity: maximum number of bits that can be stored
    :ivar int capacity: maximum number of bits that can be stored
    :ivar list packets: :class:`Packets <.Packet>` currently in storage
    """
    def __init__(self, capacity):
        self.capacity = capacity
        self.packets = []

    def add(self, packet):
        """
        Adds packet to `packets` if `capacity` will not be exceeded,
        drops packet if buffer if full.

        :param packet: :class:`.Packet` added to buffer.
        """
        current_level = sum(packet.size for packet in self.packets)
        if current_level + packet.size <= self.capacity:
            self.packets.append(packet)
        else:
            # The packet cannot be stored, so the packet is dropped
            # TODO: Insert callback for simulation monitor (report that a packet was dropped)
            pass

class Flow:
    """Representation of a connection between access points

    Flows try to transmit data from :class:`.Host` to :class:`.Host`.

    :param source: source :class:`.Host`
    :param destination: destination :class:`.Host`
    :param float amount: amount of data to transmit
    :ivar source: source :class:`.Host`
    :ivar destination: destination :class:`.Host`
    :ivar float amount: amount of data to transmit
    """
    def __init__(self, source, destination, amount):
        self.source = source
        self.destination = destination
        self.amount = amount

        self.W=1
        self.tla=StopAndWait(self)

    def __str__(self):
        return ('Flow from ' + self.source.address +
                ' to ' + self.destination.address)

    def make_packet(self, packet_num):
        """
        Make a packet based on the packet number
        """
        packet=DataPacket(packet_num, False, 0, self.source, self.destination)
        return packet

    def make_ack_packet(self, packet):
        """
        Make a ack packet
        """
        ack_packet=DataPacket(packet.number, True, 0, packet.destination, packet.source)
        return ack_packet

    def send_packet(self, packet):
        """
        When possible, TLA use this method to send a packet
        """
        self.host.send(packet)

    def receive_packet(self, packet):
        """
        If the packet is a data packet, generate an ack packet
        """
        if packet.ack==True
        """
        If the packet is a ack packet, call tla.rcv_ack()
        """

    def time_out(self):
        """
        When time out happens, run TLA
        Time_out timers should be reset if a the ack arrive
        """

    def react_to_flow_start(self, event):
        # TODO: react by sending packets to Host
        pass

class Host:
    """Representation of an access point

    Hosts send :class:`Packets <.Packet>` through a :class:`.Link` to a
    :class:`.Router` or to another :class:`.Host`.

    :param str address: IP address
    :ivar str address: IP address
    :ivar list flows: :class:`Flows <.Flow>` on this :class:`.Host`
    :ivar link: :class:`Link` connected to this :class:`.Host`
    """
    def __init__(self, address):
        self.address = address
        self.flows = []
        self.link = None

    def __str__(self):
        return 'Host at ' + self.address

    def send(self, packet):
        self.link.add(packet)

    def receive(self, packet):
        # TODO: pass to flows[packet.destination]
        pass

class Link:
    """Representation of a physical link between access points or routers

    Links carry packets from one end to the other.

    :param source: source :class:`.Host` or :class:`.Router`
    :param destination: destination :class:`.Host` or :class:`.Router`
    :param float delay: amount of time required to transmit a :class:`.Packet`
    :param float rate: speed of removing data from source
    :param int buffer_capacity: :class:`.Buffer` capacity in bits
    :ivar source: source :class:`.Host` or :class:`.Router`
    :ivar destination: destination :class:`.Host` or :class:`.Router`
    :ivar float delay: amount of time required to transmit a :class:`.Packet`
    :ivar list buffer: :class:`Packets <.Packet>` currently in transmission
    :ivar bool busy: whether currently removing data from source
    :ivar float utilization: fraction of capacity in use
    """
    def __init__(self, source, destination, delay, rate, buffer_capacity):
        self.source = source
        self.destination = destination
        self.delay = delay
        self.rate = rate
        self.buffer = Buffer(capacity=buffer_capacity)
        self.busy = False
        self.utilization = 0

    def __str__(self):
        return ('Link from ' + self.source.address +
                ' to ' + self.destination.address)

    def add(self, packet):
        if self.busy:
            self.buffer.add(packet)
        else:
            self.send(packet)

    def react_to_link_available(self, event):
        # TODO: implement the pseudo-code below
        # self.busy = False
        # if packets in buffer:
        #     self.busy = True
        #     self.send(self.buffer.get_first_packet())
        pass

    def send(self, packet):
        # TODO: implement sending by scheduling LinkAvailable and PacketReceipt
        pass

class Packet:
    """Representation of a quantum of information

    Packets carry information along the network, between :class:`Hosts <.Host>`
    or :class:`Routers <.Router>`.

    :param source: source :class:`.Host` or :class:`.Router`
    :param destination: destination :class:`.Host` or :class:`.Router`
    :param int number: sequence number
    :param acknowledgement: acknowledgement... something
    :cvar int PACKET_SIZE: size of every :class:`.Packet`, in bits
    :ivar source: source :class:`.Host` or :class:`.Router`
    :ivar destination: destination :class:`.Host` or :class:`.Router`
    :ivar int number: sequence number
    :ivar acknowledgement: acknowledgement... something
    :ivar int size: size, in bits
    :ivar str timestamp: time at which the packet was created
    """
    def __init__(self, timestamp, source, destination):
        self.timestamp = timestamp
        self.source = source
        self.destination = destination
        self.size = PACKET_SIZE

class DataPacket(Packet):
    def __init__(self, number, acknowledgement, timestamp, source, destination):
        Packet.__init__(self, timestamp, source, destination)
        self.number = number
        self.acknowledgment = acknowledgment

class RouterPacket(Packet):
    def __init__(self, timestamp, routertable, source):
        Packet.__init__(self, timestamp, source, destination = 0)
        self.routertable = routertable

class Router:
    """Representation of a router...

        Routers route packets through the network to their destination Hosts.

        :param address:IP address for router
        :param list links: all connected Links
        :param Link default_gateway: default route
        :param default_gateway: default out port if can not decide route
        :ivar list links: all connected Links
        :ivar dict table: routing table
        :ivar default_gateway: default out port if can not decide route
        """
    def __init__(self, address):
        self.address = address
        self.links = []
        self.table = {}
		self.defaul_gateway = None

    def initialize_routing_table(self, all_host_ip_addresses):
	"""
	the key of table is destination (IP_address of hosts)
	the first element in value of table is the distance between current router to final host
	the second element in value of table is where to go for next hop
	"""
		self.default_gateway = self.links[0].destination.address
		for host_ip_address in all_host_ip_addresses:
			val = float("inf"), default_gateway
			table[host_ip_address] = val

    def update_router_table(self, RouterPacket):
        """
            This function is to check every item in router table if any update.
            Implement Bellman Ford algorithm here

        """

        for destination, val in RouterPacket.routertable:
            if destination in self.table:
                if val[0] + 1 < self.table[destination]:
                    update_val = val[0] + 1, RouterPacket.source
                    self.table[destination] = update_val
            else:
                update_val = val[0] + 1, RouterPacket.source
                self.table[destination] = update_val

    def generate_communication_packet(self):
        """
            Design a sepcial packet that send the whole router table of this router to communicate with its neighbor
        """
        for l in links:
            router_packet = RouterPackect(routertable = self.table, source = self.address)
            send(link = l, packet = router_packet)

    def map_route(self, packet):
        if packet.destination in table:
            route_link = table[packet.destination]
            send(link = route_link, packet = packet)
        else:
            route_link = self.default_gateway
            send(link = route_link, packet = packet)

    def receive(self, packet):
        """
            Read packet head to tell whether is a normal packet or a update_RT_communication packet
            If it is normal packet, call map_route function
            If it is update_RT_communication packet, call update_router_table function
            """
		packet = event.value
        if isinstance(packet, DataPacket):
            map_route(packet)
        elif isinstance(packet, RouterPacket):
            update_router_table(packet)

    def send(self, link, packet):
        """
            send packet to certain link
            the packet could be normal packet to forward or communication packet to send to all links.
         """
        link.add(packet)

    def react_to_routing_table_outdated(self, event):
        self.generate_communication_packet()
<commit_msg>Simple error...<commit_after>"""
This module contains all actor definitions.

.. autosummary::

    Buffer
    Flow
    Host
    Link
    Packet
    Router

.. moduleauthor:: Lan Hongjian <lanhongjianlr@gmail.com>
.. moduleauthor:: Yamei Ou <oym111@gmail.com>
.. moduleauthor:: Samuel Richerd <dondiego152@gmail.com>
.. moduleauthor:: Jan Van Bruggen <jancvanbruggen@gmail.com>
.. moduleauthor:: Junlin Zhang <neicullyn@gmail.com>
"""

from tla_stop_and_wait import StopAndWait
from cs143sim.constants import PACKET_SIZE

class Buffer:
    """Representation of a data storage container

    Buffers store data to be linked while :class:`.Link` is busy sending data.

    :param int capacity: maximum number of bits that can be stored
    :ivar int capacity: maximum number of bits that can be stored
    :ivar list packets: :class:`Packets <.Packet>` currently in storage
    """
    def __init__(self, capacity):
        self.capacity = capacity
        self.packets = []

    def add(self, packet):
        """
        Adds packet to `packets` if `capacity` will not be exceeded,
        drops packet if buffer if full.

        :param packet: :class:`.Packet` added to buffer.
        """
        current_level = sum(packet.size for packet in self.packets)
        if current_level + packet.size <= self.capacity:
            self.packets.append(packet)
        else:
            # The packet cannot be stored, so the packet is dropped
            # TODO: Insert callback for simulation monitor (report that a packet was dropped)
            pass

class Flow:
    """Representation of a connection between access points

    Flows try to transmit data from :class:`.Host` to :class:`.Host`.

    :param source: source :class:`.Host`
    :param destination: destination :class:`.Host`
    :param float amount: amount of data to transmit
    :ivar source: source :class:`.Host`
    :ivar destination: destination :class:`.Host`
    :ivar float amount: amount of data to transmit
    """
    def __init__(self, source, destination, amount):
        self.source = source
        self.destination = destination
        self.amount = amount

        self.W=1
        self.tla=StopAndWait(self)

    def __str__(self):
        return ('Flow from ' + self.source.address +
                ' to ' + self.destination.address)

    def make_packet(self, packet_num):
        """
        Make a packet based on the packet number
        """
        packet=DataPacket(packet_num, False, 0, self.source, self.destination)
        return packet

    def make_ack_packet(self, packet):
        """
        Make a ack packet
        """
        ack_packet=DataPacket(packet.number, True, 0, packet.destination, packet.source)
        return ack_packet

    def send_packet(self, packet):
        """
        When possible, TLA use this method to send a packet
        """
        self.host.send(packet)

    def receive_packet(self, packet):
        """
        If the packet is a data packet, generate an ack packet
        """
        if packet.ack==True:
            pass
        """
        If the packet is a ack packet, call tla.rcv_ack()
        """

    def time_out(self):
        """
        When time out happens, run TLA
        Time_out timers should be reset if a the ack arrive
        """

    def react_to_flow_start(self, event):
        # TODO: react by sending packets to Host
        pass

class Host:
    """Representation of an access point

    Hosts send :class:`Packets <.Packet>` through a :class:`.Link` to a
    :class:`.Router` or to another :class:`.Host`.

    :param str address: IP address
    :ivar str address: IP address
    :ivar list flows: :class:`Flows <.Flow>` on this :class:`.Host`
    :ivar link: :class:`Link` connected to this :class:`.Host`
    """
    def __init__(self, address):
        self.address = address
        self.flows = []
        self.link = None

    def __str__(self):
        return 'Host at ' + self.address

    def send(self, packet):
        self.link.add(packet)

    def receive(self, packet):
        # TODO: pass to flows[packet.destination]
        pass

class Link:
    """Representation of a physical link between access points or routers

    Links carry packets from one end to the other.

    :param source: source :class:`.Host` or :class:`.Router`
    :param destination: destination :class:`.Host` or :class:`.Router`
    :param float delay: amount of time required to transmit a :class:`.Packet`
    :param float rate: speed of removing data from source
    :param int buffer_capacity: :class:`.Buffer` capacity in bits
    :ivar source: source :class:`.Host` or :class:`.Router`
    :ivar destination: destination :class:`.Host` or :class:`.Router`
    :ivar float delay: amount of time required to transmit a :class:`.Packet`
    :ivar list buffer: :class:`Packets <.Packet>` currently in transmission
    :ivar bool busy: whether currently removing data from source
    :ivar float utilization: fraction of capacity in use
    """
    def __init__(self, source, destination, delay, rate, buffer_capacity):
        self.source = source
        self.destination = destination
        self.delay = delay
        self.rate = rate
        self.buffer = Buffer(capacity=buffer_capacity)
        self.busy = False
        self.utilization = 0

    def __str__(self):
        return ('Link from ' + self.source.address +
                ' to ' + self.destination.address)

    def add(self, packet):
        if self.busy:
            self.buffer.add(packet)
        else:
            self.send(packet)

    def react_to_link_available(self, event):
        # TODO: implement the pseudo-code below
        # self.busy = False
        # if packets in buffer:
        #     self.busy = True
        #     self.send(self.buffer.get_first_packet())
        pass

    def send(self, packet):
        # TODO: implement sending by scheduling LinkAvailable and PacketReceipt
        pass

class Packet:
    """Representation of a quantum of information

    Packets carry information along the network, between :class:`Hosts <.Host>`
    or :class:`Routers <.Router>`.

    :param source: source :class:`.Host` or :class:`.Router`
    :param destination: destination :class:`.Host` or :class:`.Router`
    :param int number: sequence number
    :param acknowledgement: acknowledgement... something
    :cvar int PACKET_SIZE: size of every :class:`.Packet`, in bits
    :ivar source: source :class:`.Host` or :class:`.Router`
    :ivar destination: destination :class:`.Host` or :class:`.Router`
    :ivar int number: sequence number
    :ivar acknowledgement: acknowledgement... something
    :ivar int size: size, in bits
    :ivar str timestamp: time at which the packet was created
    """
    def __init__(self, timestamp, source, destination):
        self.timestamp = timestamp
        self.source = source
        self.destination = destination
        self.size = PACKET_SIZE

class DataPacket(Packet):
    def __init__(self, number, acknowledgement, timestamp, source, destination):
        Packet.__init__(self, timestamp, source, destination)
        self.number = number
        self.acknowledgment = acknowledgment

class RouterPacket(Packet):
    def __init__(self, timestamp, routertable, source):
        Packet.__init__(self, timestamp, source, destination = 0)
        self.routertable = routertable

class Router:
    """Representation of a router...

        Routers route packets through the network to their destination Hosts.

        :param address:IP address for router
        :param list links: all connected Links
        :param Link default_gateway: default route
        :param default_gateway: default out port if can not decide route
        :ivar list links: all connected Links
        :ivar dict table: routing table
        :ivar default_gateway: default out port if can not decide route
        """
    def __init__(self, address):
        self.address = address
        self.links = []
        self.table = {}
		self.defaul_gateway = None

    def initialize_routing_table(self, all_host_ip_addresses):
	"""
	the key of table is destination (IP_address of hosts)
	the first element in value of table is the distance between current router to final host
	the second element in value of table is where to go for next hop
	"""
		self.default_gateway = self.links[0].destination.address
		for host_ip_address in all_host_ip_addresses:
			val = float("inf"), default_gateway
			table[host_ip_address] = val

    def update_router_table(self, RouterPacket):
        """
            This function is to check every item in router table if any update.
            Implement Bellman Ford algorithm here

        """

        for destination, val in RouterPacket.routertable:
            if destination in self.table:
                if val[0] + 1 < self.table[destination]:
                    update_val = val[0] + 1, RouterPacket.source
                    self.table[destination] = update_val
            else:
                update_val = val[0] + 1, RouterPacket.source
                self.table[destination] = update_val

    def generate_communication_packet(self):
        """
            Design a sepcial packet that send the whole router table of this router to communicate with its neighbor
        """
        for l in links:
            router_packet = RouterPackect(routertable = self.table, source = self.address)
            send(link = l, packet = router_packet)

    def map_route(self, packet):
        if packet.destination in table:
            route_link = table[packet.destination]
            send(link = route_link, packet = packet)
        else:
            route_link = self.default_gateway
            send(link = route_link, packet = packet)

    def receive(self, packet):
        """
            Read packet head to tell whether is a normal packet or a update_RT_communication packet
            If it is normal packet, call map_route function
            If it is update_RT_communication packet, call update_router_table function
            """
		packet = event.value
        if isinstance(packet, DataPacket):
            map_route(packet)
        elif isinstance(packet, RouterPacket):
            update_router_table(packet)

    def send(self, link, packet):
        """
            send packet to certain link
            the packet could be normal packet to forward or communication packet to send to all links.
         """
        link.add(packet)

    def react_to_routing_table_outdated(self, event):
        self.generate_communication_packet()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime, timedelta

import six
from celery.exceptions import MaxRetriesExceededError
from celery.schedules import crontab
from celery.task import task
from celery.task.base import periodic_task
from celery.utils.log import get_task_logger
from django.conf import settings
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from couchdbkit import ResourceConflict, BulkSaveError
from casexml.apps.case.mock import CaseBlock

from corehq import toggles
from corehq.form_processor.exceptions import CaseNotFound
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from corehq.form_processor.models import UserArchivedRebuild
from couchforms.exceptions import UnexpectedDeletedXForm
from corehq.apps.domain.models import Domain
from django.utils.html import format_html

from dimagi.utils.couch.bulk import BulkFetchException
from dimagi.utils.logging import notify_exception
from soil import DownloadBase
from casexml.apps.case.xform import get_case_ids_from_form
from six.moves import map

logger = get_task_logger(__name__)

@task(serializer='pickle')
def bulk_upload_async(domain, user_specs, group_specs):
    from corehq.apps.users.bulkupload import create_or_update_users_and_groups
    task = bulk_upload_async
    DownloadBase.set_progress(task, 0, 100)
    results = create_or_update_users_and_groups(
        domain,
        user_specs,
        group_specs,
        task=task,
    )
    DownloadBase.set_progress(task, 100, 100)
    return {
        'messages': results
    }

@task(serializer='pickle', )
def bulk_download_users_async(domain, download_id, user_filters):
    from corehq.apps.users.bulkupload import dump_users_and_groups, GroupNameError
    errors = []
    try:
        dump_users_and_groups(
            domain,
            download_id,
            user_filters,
            bulk_download_users_async,
        )
    except GroupNameError as e:
        group_urls = [
            reverse('group_members', args=[domain, group.get_id])
            for group in e.blank_groups
        ]

        def make_link(url, i):
            return format_html(
                '{}',
                url,
                _('Blank Group %s') % i
            )

        group_links = [
            make_link(url, i + 1)
            for i, url in enumerate(group_urls)
        ]
        errors.append(format_html(
            _(
                'The following groups have no name. '
                'Please name them before continuing: {}'
            ),
            mark_safe(', '.join(group_links))
        ))
    except BulkFetchException:
        errors.append(_('Error exporting data. Please try again later.'))

    return {
        'errors': errors
    }

@task(serializer='pickle', rate_limit=2, queue='background_queue', ignore_result=True)  # limit this to two bulk saves a second so cloudant has time to reindex
def tag_cases_as_deleted_and_remove_indices(domain, case_ids, deletion_id, deletion_date):
    from corehq.apps.sms.tasks import delete_phone_numbers_for_owners
    from corehq.messaging.scheduling.tasks import delete_schedule_instances_for_cases
    CaseAccessors(domain).soft_delete_cases(list(case_ids), deletion_date, deletion_id)
    _remove_indices_from_deleted_cases_task.delay(domain, case_ids)
    delete_phone_numbers_for_owners.delay(case_ids)
    delete_schedule_instances_for_cases.delay(domain, case_ids)

@task(serializer='pickle', rate_limit=2, queue='background_queue', ignore_result=True, acks_late=True)
def tag_forms_as_deleted_rebuild_associated_cases(user_id, domain, form_id_list, deletion_id,
                                                  deletion_date, deleted_cases=None):
    """
    Upon user deletion, mark associated forms as deleted and prep cases
    for a rebuild.
    - 2 saves/sec for cloudant slowness (rate_limit)
    """
    deleted_cases = deleted_cases or set()
    cases_to_rebuild = set()

    for form in FormAccessors(domain).iter_forms(form_id_list):
        if form.domain != domain or not form.is_normal:
            continue

        # rebuild all cases anyways since we don't know if this has run or not if the task was killed
        cases_to_rebuild.update(get_case_ids_from_form(form))

    # do this after getting case_id's since iter_forms won't return deleted forms
    FormAccessors(domain).soft_delete_forms(list(form_id_list), deletion_date, deletion_id)

    detail = UserArchivedRebuild(user_id=user_id)
    for case_id in cases_to_rebuild - deleted_cases:
        _rebuild_case_with_retries.delay(domain, case_id, detail)

def _get_forms_to_modify(domain, modified_forms, modified_cases, is_deletion):
    """Used on user.retire() and user.unretire()

    Returns a list of IDs of forms which only modify the cases passed in and
    which aren't already listed in `modified_forms`.
    """
    form_ids_to_modify = set()
    for case_id in modified_cases:
        try:
            xform_ids = CaseAccessors(domain).get_case(case_id).xform_ids
        except CaseNotFound:
            xform_ids = []
        form_ids_to_modify |= set(xform_ids) - modified_forms

    def _is_safe_to_modify(form):
        if form.domain != domain:
            return False

        case_ids = get_case_ids_from_form(form)
        # all cases touched by the form and not already modified
        for case in CaseAccessors(domain).iter_cases(case_ids - modified_cases):
            if case.is_deleted != is_deletion:
                # we can't delete/undelete this form - this would change the state of `case`
                return False

        # all cases touched by this form are deleted
        return True

    if is_deletion or Domain.get_by_name(domain).use_sql_backend:
        all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify)
    else:
        # accessor.iter_forms doesn't include deleted forms on the couch backend
        all_forms = list(map(FormAccessors(domain).get_form, form_ids_to_modify))
    return [form.form_id for form in all_forms if _is_safe_to_modify(form)]

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date):
    to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True)
    FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id)

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def undelete_system_forms(domain, deleted_forms, deleted_cases):
    """The reverse of tag_system_forms_as_deleted; called on user.unretire()"""
    to_undelete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=False)
    FormAccessors(domain).soft_undelete_forms(to_undelete)

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def _remove_indices_from_deleted_cases_task(domain, case_ids):
    if toggles.SKIP_REMOVE_INDICES.enabled(domain):
        return

    # todo: we may need to add retry logic here but will wait to see
    # what errors we should be catching
    try:
        remove_indices_from_deleted_cases(domain, case_ids)
    except BulkSaveError as e:
        notify_exception(
            None,
            "_remove_indices_from_deleted_cases_task "
            "experienced a BulkSaveError. errors: {!r}".format(e.errors)
        )
        raise

def remove_indices_from_deleted_cases(domain, case_ids):
    from corehq.apps.hqcase.utils import submit_case_blocks
    deleted_ids = set(case_ids)
    indexes_referencing_deleted_cases = CaseAccessors(domain).get_all_reverse_indices_info(list(case_ids))
    case_updates = [
        CaseBlock(
            case_id=index_info.case_id,
            index={
                index_info.identifier: (index_info.referenced_type, '')  # blank string = delete index
            }
        ).as_string().decode('utf-8')
        for index_info in indexes_referencing_deleted_cases
        if index_info.case_id not in deleted_ids
    ]
    device_id = __name__ + ".remove_indices_from_deleted_cases"
    submit_case_blocks(case_updates, domain, device_id=device_id)

@task(serializer='pickle', bind=True, queue='background_queue', ignore_result=True,
      default_retry_delay=5 * 60, max_retries=3, acks_late=True)
def _rebuild_case_with_retries(self, domain, case_id, detail):
    """
    Rebuild a case with retries
    - retry in 5 min if failure occurs after (default_retry_delay)
    - retry a total of 3 times
    """
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    try:
        rebuild_case_from_forms(domain, case_id, detail)
    except (UnexpectedDeletedXForm, ResourceConflict) as exc:
        try:
            self.retry(exc=exc)
        except MaxRetriesExceededError:
            notify_exception(
                "Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id)
            )

@periodic_task(
    run_every=crontab(hour=23, minute=55),
    queue='background_queue',
)
def resend_pending_invitations():
    from corehq.apps.users.models import Invitation
    days_to_resend = (15, 29)
    days_to_expire = 30
    domains = Domain.get_all()
    for domain_obj in domains:
        invitations = Invitation.by_domain(domain_obj.name)
        for invitation in invitations:
            days = (datetime.utcnow() - invitation.invited_on).days
            if days in days_to_resend:
                invitation.send_activation_email(days_to_expire - days)

@task(serializer='pickle')
def turn_on_demo_mode_task(commcare_user_id, domain):
    from corehq.apps.ota.utils import turn_on_demo_mode
    from corehq.apps.users.models import CommCareUser

    user = CommCareUser.get(commcare_user_id)
    DownloadBase.set_progress(turn_on_demo_mode_task, 0, 100)
    results = turn_on_demo_mode(user, domain)
    DownloadBase.set_progress(turn_on_demo_mode_task, 100, 100)

    return {
        'messages': results
    }

@task(serializer='pickle')
def reset_demo_user_restore_task(commcare_user_id, domain):
    from corehq.apps.ota.utils import reset_demo_user_restore
    from corehq.apps.users.models import CommCareUser

    user = CommCareUser.get(commcare_user_id)

    DownloadBase.set_progress(reset_demo_user_restore_task, 0, 100)

    try:
        reset_demo_user_restore(user, domain)
        results = {'errors': []}
    except Exception as e:
        notify_exception(None, message=six.text_type(e))
        results = {'errors': [
            _("Something went wrong in creating restore for the user. Please try again or report an issue")
        ]}

    DownloadBase.set_progress(reset_demo_user_restore_task, 100, 100)
    return {'messages': results}

@task(serializer='pickle')
def remove_unused_custom_fields_from_users_task(domain):
    from corehq.apps.users.custom_data import remove_unused_custom_fields_from_users
    remove_unused_custom_fields_from_users(domain)

@task()
def update_domain_date(user_id, domain):
    from corehq.apps.users.models import WebUser
    user = WebUser.get_by_user_id(user_id, domain)
    domain_membership = user.get_domain_membership(domain)
    domain_membership.last_accessed = datetime.today().date()
    user.save()
<commit_msg>Check last domain access date before creating celery task<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime

import six
from celery.exceptions import MaxRetriesExceededError
from celery.schedules import crontab
from celery.task import task
from celery.task.base import periodic_task
from celery.utils.log import get_task_logger
from django.conf import settings
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from couchdbkit import ResourceConflict, BulkSaveError
from casexml.apps.case.mock import CaseBlock

from corehq import toggles
from corehq.form_processor.exceptions import CaseNotFound
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from corehq.form_processor.models import UserArchivedRebuild
from couchforms.exceptions import UnexpectedDeletedXForm
from corehq.apps.domain.models import Domain
from django.utils.html import format_html

from dimagi.utils.couch.bulk import BulkFetchException
from dimagi.utils.logging import notify_exception
from soil import DownloadBase
from casexml.apps.case.xform import get_case_ids_from_form
from six.moves import map

logger = get_task_logger(__name__)

@task(serializer='pickle')
def bulk_upload_async(domain, user_specs, group_specs):
    from corehq.apps.users.bulkupload import create_or_update_users_and_groups
    task = bulk_upload_async
    DownloadBase.set_progress(task, 0, 100)
    results = create_or_update_users_and_groups(
        domain,
        user_specs,
        group_specs,
        task=task,
    )
    DownloadBase.set_progress(task, 100, 100)
    return {
        'messages': results
    }

@task(serializer='pickle', )
def bulk_download_users_async(domain, download_id, user_filters):
    from corehq.apps.users.bulkupload import dump_users_and_groups, GroupNameError
    errors = []
    try:
        dump_users_and_groups(
            domain,
            download_id,
            user_filters,
            bulk_download_users_async,
        )
    except GroupNameError as e:
        group_urls = [
            reverse('group_members', args=[domain, group.get_id])
            for group in e.blank_groups
        ]

        def make_link(url, i):
            return format_html(
                '{}',
                url,
                _('Blank Group %s') % i
            )

        group_links = [
            make_link(url, i + 1)
            for i, url in enumerate(group_urls)
        ]
        errors.append(format_html(
            _(
                'The following groups have no name. '
                'Please name them before continuing: {}'
            ),
            mark_safe(', '.join(group_links))
        ))
    except BulkFetchException:
        errors.append(_('Error exporting data. Please try again later.'))

    return {
        'errors': errors
    }

@task(serializer='pickle', rate_limit=2, queue='background_queue', ignore_result=True)  # limit this to two bulk saves a second so cloudant has time to reindex
def tag_cases_as_deleted_and_remove_indices(domain, case_ids, deletion_id, deletion_date):
    from corehq.apps.sms.tasks import delete_phone_numbers_for_owners
    from corehq.messaging.scheduling.tasks import delete_schedule_instances_for_cases
    CaseAccessors(domain).soft_delete_cases(list(case_ids), deletion_date, deletion_id)
    _remove_indices_from_deleted_cases_task.delay(domain, case_ids)
    delete_phone_numbers_for_owners.delay(case_ids)
    delete_schedule_instances_for_cases.delay(domain, case_ids)

@task(serializer='pickle', rate_limit=2, queue='background_queue', ignore_result=True, acks_late=True)
def tag_forms_as_deleted_rebuild_associated_cases(user_id, domain, form_id_list, deletion_id,
                                                  deletion_date, deleted_cases=None):
    """
    Upon user deletion, mark associated forms as deleted and prep cases
    for a rebuild.
    - 2 saves/sec for cloudant slowness (rate_limit)
    """
    deleted_cases = deleted_cases or set()
    cases_to_rebuild = set()

    for form in FormAccessors(domain).iter_forms(form_id_list):
        if form.domain != domain or not form.is_normal:
            continue

        # rebuild all cases anyways since we don't know if this has run or not if the task was killed
        cases_to_rebuild.update(get_case_ids_from_form(form))

    # do this after getting case_id's since iter_forms won't return deleted forms
    FormAccessors(domain).soft_delete_forms(list(form_id_list), deletion_date, deletion_id)

    detail = UserArchivedRebuild(user_id=user_id)
    for case_id in cases_to_rebuild - deleted_cases:
        _rebuild_case_with_retries.delay(domain, case_id, detail)

def _get_forms_to_modify(domain, modified_forms, modified_cases, is_deletion):
    """Used on user.retire() and user.unretire()

    Returns a list of IDs of forms which only modify the cases passed in and
    which aren't already listed in `modified_forms`.
    """
    form_ids_to_modify = set()
    for case_id in modified_cases:
        try:
            xform_ids = CaseAccessors(domain).get_case(case_id).xform_ids
        except CaseNotFound:
            xform_ids = []
        form_ids_to_modify |= set(xform_ids) - modified_forms

    def _is_safe_to_modify(form):
        if form.domain != domain:
            return False

        case_ids = get_case_ids_from_form(form)
        # all cases touched by the form and not already modified
        for case in CaseAccessors(domain).iter_cases(case_ids - modified_cases):
            if case.is_deleted != is_deletion:
                # we can't delete/undelete this form - this would change the state of `case`
                return False

        # all cases touched by this form are deleted
        return True

    if is_deletion or Domain.get_by_name(domain).use_sql_backend:
        all_forms = FormAccessors(domain).iter_forms(form_ids_to_modify)
    else:
        # accessor.iter_forms doesn't include deleted forms on the couch backend
        all_forms = list(map(FormAccessors(domain).get_form, form_ids_to_modify))
    return [form.form_id for form in all_forms if _is_safe_to_modify(form)]

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def tag_system_forms_as_deleted(domain, deleted_forms, deleted_cases, deletion_id, deletion_date):
    to_delete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=True)
    FormAccessors(domain).soft_delete_forms(to_delete, deletion_date, deletion_id)

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def undelete_system_forms(domain, deleted_forms, deleted_cases):
    """The reverse of tag_system_forms_as_deleted; called on user.unretire()"""
    to_undelete = _get_forms_to_modify(domain, deleted_forms, deleted_cases, is_deletion=False)
    FormAccessors(domain).soft_undelete_forms(to_undelete)

@task(serializer='pickle', queue='background_queue', ignore_result=True, acks_late=True)
def _remove_indices_from_deleted_cases_task(domain, case_ids):
    if toggles.SKIP_REMOVE_INDICES.enabled(domain):
        return

    # todo: we may need to add retry logic here but will wait to see
    # what errors we should be catching
    try:
        remove_indices_from_deleted_cases(domain, case_ids)
    except BulkSaveError as e:
        notify_exception(
            None,
            "_remove_indices_from_deleted_cases_task "
            "experienced a BulkSaveError. errors: {!r}".format(e.errors)
        )
        raise

def remove_indices_from_deleted_cases(domain, case_ids):
    from corehq.apps.hqcase.utils import submit_case_blocks
    deleted_ids = set(case_ids)
    indexes_referencing_deleted_cases = CaseAccessors(domain).get_all_reverse_indices_info(list(case_ids))
    case_updates = [
        CaseBlock(
            case_id=index_info.case_id,
            index={
                index_info.identifier: (index_info.referenced_type, '')  # blank string = delete index
            }
        ).as_string().decode('utf-8')
        for index_info in indexes_referencing_deleted_cases
        if index_info.case_id not in deleted_ids
    ]
    device_id = __name__ + ".remove_indices_from_deleted_cases"
    submit_case_blocks(case_updates, domain, device_id=device_id)

@task(serializer='pickle', bind=True, queue='background_queue', ignore_result=True,
      default_retry_delay=5 * 60, max_retries=3, acks_late=True)
def _rebuild_case_with_retries(self, domain, case_id, detail):
    """
    Rebuild a case with retries
    - retry in 5 min if failure occurs after (default_retry_delay)
    - retry a total of 3 times
    """
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    try:
        rebuild_case_from_forms(domain, case_id, detail)
    except (UnexpectedDeletedXForm, ResourceConflict) as exc:
        try:
            self.retry(exc=exc)
        except MaxRetriesExceededError:
            notify_exception(
                "Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id)
            )

@periodic_task(
    run_every=crontab(hour=23, minute=55),
    queue='background_queue',
)
def resend_pending_invitations():
    from corehq.apps.users.models import Invitation
    days_to_resend = (15, 29)
    days_to_expire = 30
    domains = Domain.get_all()
    for domain_obj in domains:
        invitations = Invitation.by_domain(domain_obj.name)
        for invitation in invitations:
            days = (datetime.utcnow() - invitation.invited_on).days
            if days in days_to_resend:
                invitation.send_activation_email(days_to_expire - days)

@task(serializer='pickle')
def turn_on_demo_mode_task(commcare_user_id, domain):
    from corehq.apps.ota.utils import turn_on_demo_mode
    from corehq.apps.users.models import CommCareUser

    user = CommCareUser.get(commcare_user_id)
    DownloadBase.set_progress(turn_on_demo_mode_task, 0, 100)
    results = turn_on_demo_mode(user, domain)
    DownloadBase.set_progress(turn_on_demo_mode_task, 100, 100)

    return {
        'messages': results
    }

@task(serializer='pickle')
def reset_demo_user_restore_task(commcare_user_id, domain):
    from corehq.apps.ota.utils import reset_demo_user_restore
    from corehq.apps.users.models import CommCareUser

    user = CommCareUser.get(commcare_user_id)

    DownloadBase.set_progress(reset_demo_user_restore_task, 0, 100)

    try:
        reset_demo_user_restore(user, domain)
        results = {'errors': []}
    except Exception as e:
        notify_exception(None, message=six.text_type(e))
        results = {'errors': [
            _("Something went wrong in creating restore for the user. Please try again or report an issue")
        ]}

    DownloadBase.set_progress(reset_demo_user_restore_task, 100, 100)
    return {'messages': results}

@task(serializer='pickle')
def remove_unused_custom_fields_from_users_task(domain):
    from corehq.apps.users.custom_data import remove_unused_custom_fields_from_users
    remove_unused_custom_fields_from_users(domain)

@task()
def update_domain_date(user_id, domain):
    from corehq.apps.users.models import WebUser
    user = WebUser.get_by_user_id(user_id, domain)
    domain_membership = user.get_domain_membership(domain)
    domain_membership.last_accessed = datetime.today().date()
<|endoftext|>