<commit_before># Copyright (c) 2017 Fortinet, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

#    FortiAuthenticator API request format templates.

# About api request message naming regulations:
# Prefix         HTTP method
# ADD_XXX      -->    POST
# SET_XXX      -->    PUT
# DELETE_XXX   -->    DELETE
# GET_XXX      -->    GET
# MODIFY_XXX   -->    PATCH

# usergroups
# query usergroups
GET_USERGROUPS = """
{
    {% if id is defined %}
        "path": "/api/v1/usergroups/{{ id }}/",
    {% else %}
        {% if name is defined %}
            "path": "/api/v1/usergroups/?name={{ name }}",
        {% else %}
            "path": "/api/v1/usergroups/",
        {% endif %}
    {% endif %}
    "method": "GET"
}
"""

# create an usergroup
CREATE_USERGROUP = """
{
    "path": "/api/v1/usergroups/",
    "method": "POST",
    "body": {
        "name": "{{ name }}"
    }
}
"""

# modify an usergroup
MODIFY_USERGROUP = """
{
    "path": "/api/v1/usergroups/{{ id }}/",
    "method": "PATCH",
     "body": {
        {% if users is defined %}
            "users": ["{{ users }}"]
        {% endif %}
     }
}
"""

# delete an usergroup
DELETE_USERGROUP = """
{
    "path": "/api/v1/usergroups/",
    "method": "DELETE",
    "body": {
        "name": "{{ name }}"
    }
}
"""

## user
# query users
GET_USERS = """
{
    {% if id is defined %}
        "path": "/api/v1/localusers/{{ id }}/",
    {% else %}
        {% set _options = {
            "username": username,
            "token_auth": token_auth,
            "ftk_only": ftk_only,
            "ftm_act_method": ftm_act_method,
            "token_type": token_type,
            "token_serial": token_serial,
            "first_name": first_name,
            "last_name": last_name,
            "user_groups": user_groups,
            "address": address,
            "city": city,
            "state": state,
            "country": country,
            "email": email,
            "mobile_number": mobile_number,
            "phone_number": phone_number,
            "expires_at": expires_at,
            "custom1": custom1,
            "custom2": custom2,
            "custom3": custom3,
            "active": active
        } %}
        {% set _query = [] %}
        {% for k, v in _options.iteritems() if v is defined and v %}
            {% if _query.append('&'+k+'='+v) %}
            {% endif %}
        {% endfor %}
        {% if _query %}
            {% set _query = ''.join(_query) %}
            "path": "/api/v1/localusers/?{{ _query }}",
        {% else %}
            "path": "/api/v1/localusers/",
        {% endif %}
    {% endif %}
    "method": "GET"
}
"""

# create an user
CREATE_USER = """
{
    "path": "/api/v1/localusers/",
    "method": "POST",
    "body": {
        {%
            set _options = {
                "token_auth": token_auth,
                "ftk_only": ftk_only,
                "ftm_act_method": ftm_act_method,
                "token_type": token_type,
                "token_serial": token_serial,
                "first_name": first_name,
                "last_name": last_name,
                "address": address,
                "city": city,
                "state": state,
                "country": country,
                "email": email,
                "mobile_number": mobile_number,
                "phone_number": phone_number,
                "expires_at": expires_at,
                "custom1": custom1,
                "custom2": custom2,
                "custom3": custom3,
                "active": active
            }
        %}
        {% if user_groups is defined %}
            "user_groups": ["{{ user_groups }}"],
        {% endif %}
        {% for k, v in _options.iteritems() if v is defined and v %}
            "{{ k }}": "{{ v }}",
        {% endfor %}
        "username": "{{ username }}"
    }
}
"""

# modify an user
MODIFY_USER = """
{
    "path": "/api/v1/localusers/{{ id }}/",
    "method": "PATCH",
    "body": {
        {%
            set _options = {
                "token_auth": token_auth,
                "ftk_only": ftk_only,
                "ftm_act_method": ftm_act_method,
                "token_type": token_type,
                "token_serial": token_serial,
                "first_name": first_name,
                "last_name": last_name,
                "user_groups": user_groups,
                "address": address,
                "city": city,
                "state": state,
                "country": country,
                "email": email,
                "mobile_number": mobile_number,
                "phone_number": phone_number,
                "expires_at": expires_at,
                "custom1": custom1,
                "custom2": custom2,
                "custom3": custom3,
                "active": active
            }
        %}
        {% for k, v in _options.iteritems() if v is defined and v is not None %}
            "{{ k }}": "{{ v }}",
        {% endfor %}
        "id": {{ id }}
    }
}
"""

# delete an usergroup
DELETE_USER = """
{
    "path": "/api/v1/localusers/{{ id }}/",
    "method": "DELETE"
}
"""

## FortiToken
# query FortiToken
GET_FORTITOKENS = """
{
    {% set _options = {
        "serial": serial,
        "status": status,
        "type": type,
        "limit": limit
    } %}
    {% set _query = [] %}
    {% for k, v in _options.iteritems() if v is defined and v %}
        {% if _query.append('&'+k+'='+v) %}
        {% endif %}
    {% endfor %}
    {% if _query %}
        {% set _query = ''.join(_query) %}
        {% set _query = '?' + _query[1:] %}
        "path": "/api/v1/fortitokens/{{ _query }}",
    {% else %}
        "path": "/api/v1/fortitokens/",
    {% endif %}
    "method": "GET"
}
"""

# user authentication
CREATE_AUTH = """
{
    "path": "/api/v1/auth/",
    "method": "POST",
    "body": {
        {% if token_code is defined %}
            "token_code": "{{ token_code }}",
        {% endif %}
        {% if password is defined %}
            "password": "{{ password }}",
        {% endif %}
        "username": "{{ username }}"
    }
}
"""
<commit_msg>fixed several minimum bugs of the fortiauth api client<commit_after># Copyright (c) 2017 Fortinet, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

#    FortiAuthenticator API request format templates.

# About api request message naming regulations:
# Prefix         HTTP method
# ADD_XXX      -->    POST
# SET_XXX      -->    PUT
# DELETE_XXX   -->    DELETE
# GET_XXX      -->    GET
# MODIFY_XXX   -->    PATCH

# usergroups
# query usergroups
GET_USERGROUPS = """
{
    {% if id is defined %}
        "path": "/api/v1/usergroups/{{ id }}/",
    {% else %}
        {% if name is defined %}
            "path": "/api/v1/usergroups/?name={{ name }}",
        {% else %}
            "path": "/api/v1/usergroups/",
        {% endif %}
    {% endif %}
    "method": "GET"
}
"""

# create an usergroup
CREATE_USERGROUP = """
{
    "path": "/api/v1/usergroups/",
    "method": "POST",
    "body": {
        "name": "{{ name }}"
    }
}
"""

# modify an usergroup
MODIFY_USERGROUP = """
{
    "path": "/api/v1/usergroups/{{ id }}/",
    "method": "PATCH",
     "body": {
        {% if users is defined %}
            "users": ["{{ users }}"]
        {% endif %}
     }
}
"""

# delete an usergroup
DELETE_USERGROUP = """
{
    "path": "/api/v1/usergroups/",
    "method": "DELETE",
    "body": {
        "name": "{{ name }}"
    }
}
"""

## user
# query users
GET_USERS = """
{
    {% if id is defined %}
        "path": "/api/v1/localusers/{{ id }}/",
    {% else %}
        {% set _options = {
            "username": username,
            "token_auth": token_auth,
            "ftk_only": ftk_only,
            "ftm_act_method": ftm_act_method,
            "token_type": token_type,
            "token_serial": token_serial,
            "first_name": first_name,
            "last_name": last_name,
            "user_groups": user_groups,
            "address": address,
            "city": city,
            "state": state,
            "country": country,
            "email": email,
            "mobile_number": mobile_number,
            "phone_number": phone_number,
            "expires_at": expires_at,
            "custom1": custom1,
            "custom2": custom2,
            "custom3": custom3,
            "active": active
        } %}
        {% set _query = [] %}
        {% for k, v in _options.iteritems() if v is defined %}
            {% if _query.append('&'+k+'='+v) %}
            {% endif %}
        {% endfor %}
        {% if _query %}
            {% set _query = ''.join(_query) %}
            "path": "/api/v1/localusers/?{{ _query }}",
        {% else %}
            "path": "/api/v1/localusers/",
        {% endif %}
    {% endif %}
    "method": "GET"
}
"""

# create an user
CREATE_USER = """
{
    "path": "/api/v1/localusers/",
    "method": "POST",
    "body": {
        {%
            set _options = {
                "token_auth": token_auth,
                "ftk_only": ftk_only,
                "ftm_act_method": ftm_act_method,
                "token_type": token_type,
                "token_serial": token_serial,
                "first_name": first_name,
                "last_name": last_name,
                "address": address,
                "city": city,
                "state": state,
                "country": country,
                "email": email,
                "mobile_number": mobile_number,
                "phone_number": phone_number,
                "expires_at": expires_at,
                "custom1": custom1,
                "custom2": custom2,
                "custom3": custom3,
                "active": active
            }
        %}
        {% if user_groups is defined %}
            "user_groups": ["{{ user_groups }}"],
        {% endif %}
        {% for k, v in _options.iteritems() if v is defined %}
            "{{ k }}": "{{ v }}",
        {% endfor %}
        "username": "{{ username }}"
    }
}
"""

# modify an user
MODIFY_USER = """
{
    "path": "/api/v1/localusers/{{ id }}/",
    "method": "PATCH",
    "body": {
        {%
            set _options = {
                "token_auth": token_auth,
                "ftk_only": ftk_only,
                "ftm_act_method": ftm_act_method,
                "token_type": token_type,
                "token_serial": token_serial,
                "first_name": first_name,
                "last_name": last_name,
                "user_groups": user_groups,
                "address": address,
                "city": city,
                "state": state,
                "country": country,
                "email": email,
                "mobile_number": mobile_number,
                "phone_number": phone_number,
                "expires_at": expires_at,
                "custom1": custom1,
                "custom2": custom2,
                "custom3": custom3,
                "active": active
            }
        %}
        {% for k, v in _options.iteritems() if v is defined %}
            "{{ k }}": "{{ v }}",
        {% endfor %}
        "id": {{ id }}
    }
}
"""

# delete an usergroup
DELETE_USER = """
{
    "path": "/api/v1/localusers/{{ id }}/",
    "method": "DELETE"
}
"""

## FortiToken
# query FortiToken
GET_FORTITOKENS = """
{
    {% set _options = {
        "serial": serial,
        "status": status,
        "type": type,
        "limit": limit
    } %}
    {% set _query = [] %}
    {% for k, v in _options.iteritems() if v is defined %}
        {% if _query.append('&'+k+'='+v) %}
        {% endif %}
    {% endfor %}
    {% if _query %}
        {% set _query = ''.join(_query) %}
        {% set _query = '?' + _query[1:] %}
        "path": "/api/v1/fortitokens/{{ _query }}",
    {% else %}
        "path": "/api/v1/fortitokens/",
    {% endif %}
    "method": "GET"
}
"""

# user authentication
CREATE_AUTH = """
{
    "path": "/api/v1/auth/",
    "method": "POST",
    "body": {
        {% if token_code is defined %}
            "token_code": "{{ token_code }}",
        {% endif %}
        {% if password is defined %}
            "password": "{{ password }}",
        {% endif %}
        "username": "{{ username }}"
    }
}
"""
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import os.path as op

from pynodegl_utils.com import query_subproc, query_inplace

def serialize(dirname, subproc=False):
    module_pkg = 'pynodegl_utils.examples'
    if subproc:
        ret = query_subproc(query='list', pkg=module_pkg)
    else:
        ret = query_inplace(query='list', pkg=module_pkg)
    assert 'error' not in ret
    scenes = ret['scenes']

    try:
        os.makedirs(dirname)
    except OSError:
        pass

    for module_name, sub_scenes in scenes:
        for scene_name, widgets_specs in sub_scenes:
            cfg = {
                'pkg': module_pkg,
                'scene': (module_name, scene_name),
            }
            if subproc:
                ret = query_subproc(query='scene', **cfg)
            else:
                ret = query_inplace(query='scene', **cfg)
            assert 'error' not in ret
            fname = op.join(dirname, '%s_%s.ngl' % (module_name, scene_name))
            print(fname)
            open(fname, 'w').write(ret['scene'])

if __name__ == '__main__':
    import sys
    serialize(op.join(sys.argv[1]))
<commit_msg>tests: simplify check before calling makedirs<commit_after>#!/usr/bin/env python

import os
import os.path as op

from pynodegl_utils.com import query_subproc, query_inplace

def serialize(dirname, subproc=False):
    module_pkg = 'pynodegl_utils.examples'
    if subproc:
        ret = query_subproc(query='list', pkg=module_pkg)
    else:
        ret = query_inplace(query='list', pkg=module_pkg)
    assert 'error' not in ret
    scenes = ret['scenes']

    if not op.exists(dirname):
        os.makedirs(dirname)

    for module_name, sub_scenes in scenes:
        for scene_name, widgets_specs in sub_scenes:
            cfg = {
                'pkg': module_pkg,
                'scene': (module_name, scene_name),
            }
            if subproc:
                ret = query_subproc(query='scene', **cfg)
            else:
                ret = query_inplace(query='scene', **cfg)
            assert 'error' not in ret
            fname = op.join(dirname, '%s_%s.ngl' % (module_name, scene_name))
            print(fname)
            open(fname, 'w').write(ret['scene'])

if __name__ == '__main__':
    import sys
    serialize(op.join(sys.argv[1]))
<|endoftext|>


<commit_before>'''
Get system running status including CPU load, memory usage, network traffic.
'''

class SystemStatus(object):

    def __init__(self):
        pass

    def getStatus(self):
        test = {
            'cpuload': 10
        }
        return test

    def refreshCpuLoad(self):
        pass

    def refreshMemoryInfo(self):
        pass

    def refreshDiskInfo(self):
        pass

    def refreshNetworkTraffic(self):
        pass
<commit_msg>Implement the API to get system running status<commit_after>'''
Get system running status including CPU load, memory usage, network traffic.
'''
import psutil
import time

class SystemStatus(object):

    def __init__(self):
        self.timestamp = time.time()
        self.cpu_load = []
        self.mem = dict(total = 0,
                        available = 0,
                        free = 0,
                        cached = 0,
                        buffers = 0)
        self.disk_partitions = {}
        self.network = {}

        partitions = psutil.disk_partitions()
        for p in partitions:
            if p.fstype == 'ext4':
                usage = psutil.disk_usage(p.mountpoint)
                self.disk_partitions[p.mountpoint] = {
                    'total': usage.total,
                    'used': usage.used
                }

    def getStatus(self):
        timestamp = time.time()
        if (timestamp > self.timestamp + 0.8):
            self.timestamp = timestamp
            self.refreshCpuLoad()
            self.refreshMemoryInfo()
            self.refreshDiskInfo()
            self.refreshNetworkTraffic()

        result = {
            'cpu_load': self.cpu_load,
            'mem': self.mem,
            'disk': self.disk_partitions,
            'network': self.network
        }
        return result

    def refreshCpuLoad(self):
        self.cpu_load = map(int, psutil.cpu_percent(percpu=True))

    def refreshMemoryInfo(self):
        mem = psutil.virtual_memory()
        self.mem['total'] = mem.total
        self.mem['available'] = mem.available
        self.mem['free'] = mem.free
        self.mem['cached'] = mem.cached
        self.mem['buffers'] = mem.buffers

    def refreshDiskInfo(self):
        for key, value in self.disk_partitions.iteritems():
            usage = psutil.disk_usage(key)
            self.disk_partitions[key]['total'] = usage.total
            self.disk_partitions[key]['used'] = usage.used

    def refreshNetworkTraffic(self):
        excluded_interfaces = set(["lo", 'br-lan', 'docker0', 'wlan0'])
        interfaces = {}

        stats = psutil.net_if_stats()
        for key, value in stats.iteritems():
            if key in excluded_interfaces:
                continue

            interfaces[key] = {
                'isup': value.isup,
                'speed': value.speed,
                'mtu': value.mtu
            }

        addresses = psutil.net_if_addrs()
        for key, value in addresses.iteritems():
            if key in excluded_interfaces:
                continue
            for i in value:
                if i.family == 2:
                    interfaces[key]['ipv4'] = i.address
                    interfaces[key]['netmask'] = i.netmask
                elif i.family == 17:
                    interfaces[key]['mac'] = i.address

        traffic = psutil.net_io_counters(pernic=True)
        for key, value in traffic.iteritems():
            if key in excluded_interfaces:
                continue

            interfaces[key]['bytes_sent'] = value.bytes_sent
            interfaces[key]['bytes_recv'] = value.bytes_recv
            interfaces[key]['packets_sent'] = value.packets_sent
            interfaces[key]['packets_recv'] = value.packets_recv
            interfaces[key]['errin'] = value.errin
            interfaces[key]['errout'] = value.errout
            interfaces[key]['dropin'] = value.dropin
            interfaces[key]['dropout'] = value.dropout

        self.network = interfaces
<|endoftext|>


<commit_before>import json
from mock import Mock, MagicMock, ANY
import unittest

from xmodule.openendedchild import OpenEndedChild
from xmodule.open_ended_module import OpenEndedModule
from xmodule.combined_open_ended_modulev1 import CombinedOpenEndedV1Module

from xmodule.modulestore import Location
from lxml import etree
import capa.xqueue_interface as xqueue_interface
from datetime import datetime

from . import test_system
"""
Tests for the various pieces of the CombinedOpenEndedGrading system

OpenEndedChild
OpenEndedModule

"""

class OpenEndedChildTest(unittest.TestCase):
    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = json.dumps({'attempts': '10'})
    prompt = etree.XML("<prompt>This is a question prompt</prompt>")
    rubric = '''<rubric><rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric></rubric>'''
    max_score = 4

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload': False,
            }
    definition = Mock()
    descriptor = Mock()

    def setUp(self):
        self.openendedchild = OpenEndedChild(test_system, self.location,
                self.definition, self.descriptor, self.static_data, self.metadata)

    def test_latest_answer_empty(self):
        answer = self.openendedchild.latest_answer()
        self.assertEqual(answer, "")

    def test_latest_score_empty(self):
        answer = self.openendedchild.latest_score()
        self.assertEqual(answer, None)

    def test_latest_post_assessment_empty(self):
        answer = self.openendedchild.latest_post_assessment(test_system)
        self.assertEqual(answer, "")

    def test_new_history_entry(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        answer = self.openendedchild.latest_answer()
        self.assertEqual(answer, new_answer)

        new_answer = "Newer Answer"
        self.openendedchild.new_history_entry(new_answer)
        answer = self.openendedchild.latest_answer()
        self.assertEqual(new_answer, answer)

    def test_record_latest_score(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        new_score = 3
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.latest_score()
        self.assertEqual(score, 3)

        new_score = 4
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.latest_score()
        self.assertEqual(score, 4)

    def test_record_latest_post_assessment(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)

        post_assessment = "Post assessment"
        self.openendedchild.record_latest_post_assessment(post_assessment)
        self.assertEqual(post_assessment,
                self.openendedchild.latest_post_assessment(test_system))

    def test_get_score(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)

        score = self.openendedchild.get_score()
        self.assertEqual(score['score'], 0)
        self.assertEqual(score['total'], self.static_data['max_score'])

        new_score = 4
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.get_score()
        self.assertEqual(score['score'], new_score)
        self.assertEqual(score['total'], self.static_data['max_score'])

    def test_reset(self):
        self.openendedchild.reset(test_system)
        state = json.loads(self.openendedchild.get_instance_state())
        self.assertEqual(state['state'], OpenEndedChild.INITIAL)

    def test_is_last_response_correct(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(self.static_data['max_score'])
        self.assertEqual(self.openendedchild.is_last_response_correct(),
                'correct')

        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(0)
        self.assertEqual(self.openendedchild.is_last_response_correct(),
                'incorrect')

class OpenEndedModuleTest(unittest.TestCase):
    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = json.dumps({'attempts': '10'})
    prompt = etree.XML("<prompt>This is a question prompt</prompt>")
    rubric = etree.XML('''<rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric>''')
    max_score = 4

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload': False,
            'rewrite_content_links' : "",
            }

    oeparam = etree.XML('''
      <openendedparam>
            <initial_display>Enter essay here.</initial_display>
            <answer_display>This is the answer.</answer_display>
            <grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
        </openendedparam>
    ''')
    definition = {'oeparam': oeparam}
    descriptor = Mock()

    def setUp(self):
        test_system.location = self.location
        self.mock_xqueue = MagicMock()
        self.mock_xqueue.send_to_queue.return_value = (None, "Message")
        test_system.xqueue = {'interface': self.mock_xqueue, 'callback_url': '/', 'default_queuename': 'testqueue', 'waittime': 1}
        self.openendedmodule = OpenEndedModule(test_system, self.location,
                self.definition, self.descriptor, self.static_data, self.metadata)

    def test_message_post(self):
        get = {'feedback': 'feedback text',
                'submission_id': '1',
                'grader_id': '1',
                'score': 3}
        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
        student_info = {'anonymous_student_id': test_system.anonymous_student_id,
                'submission_time': qtime}
        contents = {
                'feedback': get['feedback'],
                'submission_id': int(get['submission_id']),
                'grader_id': int(get['grader_id']),
                'score': get['score'],
                'student_info': json.dumps(student_info)
                }

        result = self.openendedmodule.message_post(get, test_system)
        self.assertTrue(result['success'])
        # make sure it's actually sending something we want to the queue
        self.mock_xqueue.send_to_queue.assert_called_with(body=json.dumps(contents), header=ANY)

        state = json.loads(self.openendedmodule.get_instance_state())
        self.assertIsNotNone(state['state'], OpenEndedModule.DONE)

    def test_send_to_grader(self):
        submission = "This is a student submission"
        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
        student_info = {'anonymous_student_id': test_system.anonymous_student_id,
                'submission_time': qtime}
        contents = self.openendedmodule.payload.copy()
        contents.update({
            'student_info': json.dumps(student_info),
            'student_response': submission,
            'max_score': self.max_score
            })
        result = self.openendedmodule.send_to_grader(submission, test_system)
        self.assertTrue(result)
        self.mock_xqueue.send_to_queue.assert_called_with(body=json.dumps(contents), header=ANY)

    def update_score_single(self):
        self.openendedmodule.new_history_entry("New Entry")
        score_msg = {
                'correct': True,
                'score': 4,
                'msg': 'Grader Message',
                'feedback': "Grader Feedback"
                }
        get = {'queuekey': "abcd",
                'xqueue_body': score_msg}
        self.openendedmodule.update_score(get, test_system)

    def update_score_single(self):
        self.openendedmodule.new_history_entry("New Entry")
        feedback = {
                "success": True,
                "feedback": "Grader Feedback"
                }
        score_msg = {
                'correct': True,
                'score': 4,
                'msg': 'Grader Message',
                'feedback': json.dumps(feedback),
                'grader_type': 'IN',
                'grader_id': '1',
                'submission_id': '1',
                'success': True,
                'rubric_scores': [0],
                'rubric_scores_complete': True,
                'rubric_xml': etree.tostring(self.rubric)
                }
        get = {'queuekey': "abcd",
                'xqueue_body': json.dumps(score_msg)}
        self.openendedmodule.update_score(get, test_system)

    def test_latest_post_assessment(self):
        self.update_score_single()
        assessment = self.openendedmodule.latest_post_assessment(test_system)
        self.assertFalse(assessment == '')
        # check for errors
        self.assertFalse('errors' in assessment)

    def test_update_score(self):
        self.update_score_single()
        score = self.openendedmodule.latest_score()
        self.assertEqual(score, 4)

class CombinedOpenEndedModuleTest(unittest.TestCase):
    location = Location(["i4x", "edX", "open_ended", "combinedopenended",
                         "SampleQuestion"])

    prompt = "<prompt>This is a question prompt</prompt>"
    rubric = '''<rubric><rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric></rubric>'''
    max_score = 3

    metadata = {'attempts': '10', 'max_score': max_score}

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload' : False,
            'rewrite_content_links' : "",
            }

    oeparam = etree.XML('''
      <openendedparam>
            <initial_display>Enter essay here.</initial_display>
            <answer_display>This is the answer.</answer_display>
            <grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
        </openendedparam>
    ''')

    task_xml1 = '''
                <selfassessment>
                    <hintprompt>
                        What hint about this problem would you give to someone?
                    </hintprompt>
                    <submitmessage>
                        Save Succcesful.  Thanks for participating!
                    </submitmessage>
                </selfassessment>
            '''
    task_xml2 = '''
    <openended min_score_to_attempt="1" max_score_to_attempt="1">
		        <openendedparam>
		      		<initial_display>Enter essay here.</initial_display>
		      		<answer_display>This is the answer.</answer_display>
		      		<grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
		    	</openendedparam>
		</openended>'''
    definition = {'prompt': etree.XML(prompt), 'rubric': etree.XML(rubric), 'task_xml': [task_xml1, task_xml2]}
    descriptor = Mock()

    def setUp(self):
        self.combinedoe = CombinedOpenEndedV1Module(test_system, self.location, self.definition, self.descriptor, self.static_data, metadata=self.metadata)

    def test_get_tag_name(self):
        name = self.combinedoe.get_tag_name("<t>Tag</t>")
        self.assertEqual(name, "t")

    def test_get_last_response(self):
        response_dict = self.combinedoe.get_last_response(0)
        self.assertEqual(response_dict['type'], "selfassessment")
        self.assertEqual(response_dict['max_score'], self.max_score)
        self.assertEqual(response_dict['state'], CombinedOpenEndedV1Module.INITIAL)

    def test_update_task_states(self):
        changed = self.combinedoe.update_task_states()
        self.assertFalse(changed)

        current_task = self.combinedoe.current_task
        current_task.change_state(CombinedOpenEndedV1Module.DONE)
        changed = self.combinedoe.update_task_states()

        self.assertTrue(changed)
<commit_msg>Fix oe version tests<commit_after>import json
from mock import Mock, MagicMock, ANY
import unittest

from xmodule.openendedchild import OpenEndedChild
from xmodule.open_ended_module import OpenEndedModule
from xmodule.combined_open_ended_modulev1 import CombinedOpenEndedV1Module

from xmodule.modulestore import Location
from lxml import etree
import capa.xqueue_interface as xqueue_interface
from datetime import datetime

from . import test_system
"""
Tests for the various pieces of the CombinedOpenEndedGrading system

OpenEndedChild
OpenEndedModule

"""

class OpenEndedChildTest(unittest.TestCase):
    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = json.dumps({'attempts': '10'})
    prompt = etree.XML("<prompt>This is a question prompt</prompt>")
    rubric = '''<rubric><rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric></rubric>'''
    max_score = 4

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload': False,
            }
    definition = Mock()
    descriptor = Mock()

    def setUp(self):
        self.openendedchild = OpenEndedChild(test_system, self.location,
                self.definition, self.descriptor, self.static_data, self.metadata)

    def test_latest_answer_empty(self):
        answer = self.openendedchild.latest_answer()
        self.assertEqual(answer, "")

    def test_latest_score_empty(self):
        answer = self.openendedchild.latest_score()
        self.assertEqual(answer, None)

    def test_latest_post_assessment_empty(self):
        answer = self.openendedchild.latest_post_assessment(test_system)
        self.assertEqual(answer, "")

    def test_new_history_entry(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        answer = self.openendedchild.latest_answer()
        self.assertEqual(answer, new_answer)

        new_answer = "Newer Answer"
        self.openendedchild.new_history_entry(new_answer)
        answer = self.openendedchild.latest_answer()
        self.assertEqual(new_answer, answer)

    def test_record_latest_score(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        new_score = 3
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.latest_score()
        self.assertEqual(score, 3)

        new_score = 4
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.latest_score()
        self.assertEqual(score, 4)

    def test_record_latest_post_assessment(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)

        post_assessment = "Post assessment"
        self.openendedchild.record_latest_post_assessment(post_assessment)
        self.assertEqual(post_assessment,
                self.openendedchild.latest_post_assessment(test_system))

    def test_get_score(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)

        score = self.openendedchild.get_score()
        self.assertEqual(score['score'], 0)
        self.assertEqual(score['total'], self.static_data['max_score'])

        new_score = 4
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(new_score)
        score = self.openendedchild.get_score()
        self.assertEqual(score['score'], new_score)
        self.assertEqual(score['total'], self.static_data['max_score'])

    def test_reset(self):
        self.openendedchild.reset(test_system)
        state = json.loads(self.openendedchild.get_instance_state())
        self.assertEqual(state['state'], OpenEndedChild.INITIAL)

    def test_is_last_response_correct(self):
        new_answer = "New Answer"
        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(self.static_data['max_score'])
        self.assertEqual(self.openendedchild.is_last_response_correct(),
                'correct')

        self.openendedchild.new_history_entry(new_answer)
        self.openendedchild.record_latest_score(0)
        self.assertEqual(self.openendedchild.is_last_response_correct(),
                'incorrect')

class OpenEndedModuleTest(unittest.TestCase):
    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = json.dumps({'attempts': '10'})
    prompt = etree.XML("<prompt>This is a question prompt</prompt>")
    rubric = etree.XML('''<rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric>''')
    max_score = 4

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload': False,
            'rewrite_content_links' : "",
            }

    oeparam = etree.XML('''
      <openendedparam>
            <initial_display>Enter essay here.</initial_display>
            <answer_display>This is the answer.</answer_display>
            <grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
        </openendedparam>
    ''')
    definition = {'oeparam': oeparam}
    descriptor = Mock()

    def setUp(self):
        test_system.location = self.location
        self.mock_xqueue = MagicMock()
        self.mock_xqueue.send_to_queue.return_value = (None, "Message")
        test_system.xqueue = {'interface': self.mock_xqueue, 'callback_url': '/', 'default_queuename': 'testqueue', 'waittime': 1}
        self.openendedmodule = OpenEndedModule(test_system, self.location,
                self.definition, self.descriptor, self.static_data, self.metadata)

    def test_message_post(self):
        get = {'feedback': 'feedback text',
                'submission_id': '1',
                'grader_id': '1',
                'score': 3}
        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
        student_info = {'anonymous_student_id': test_system.anonymous_student_id,
                'submission_time': qtime}
        contents = {
                'feedback': get['feedback'],
                'submission_id': int(get['submission_id']),
                'grader_id': int(get['grader_id']),
                'score': get['score'],
                'student_info': json.dumps(student_info)
                }

        result = self.openendedmodule.message_post(get, test_system)
        self.assertTrue(result['success'])
        # make sure it's actually sending something we want to the queue
        self.mock_xqueue.send_to_queue.assert_called_with(body=json.dumps(contents), header=ANY)

        state = json.loads(self.openendedmodule.get_instance_state())
        self.assertIsNotNone(state['state'], OpenEndedModule.DONE)

    def test_send_to_grader(self):
        submission = "This is a student submission"
        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
        student_info = {'anonymous_student_id': test_system.anonymous_student_id,
                'submission_time': qtime}
        contents = self.openendedmodule.payload.copy()
        contents.update({
            'student_info': json.dumps(student_info),
            'student_response': submission,
            'max_score': self.max_score
            })
        result = self.openendedmodule.send_to_grader(submission, test_system)
        self.assertTrue(result)
        self.mock_xqueue.send_to_queue.assert_called_with(body=json.dumps(contents), header=ANY)

    def update_score_single(self):
        self.openendedmodule.new_history_entry("New Entry")
        score_msg = {
                'correct': True,
                'score': 4,
                'msg': 'Grader Message',
                'feedback': "Grader Feedback"
                }
        get = {'queuekey': "abcd",
                'xqueue_body': score_msg}
        self.openendedmodule.update_score(get, test_system)

    def update_score_single(self):
        self.openendedmodule.new_history_entry("New Entry")
        feedback = {
                "success": True,
                "feedback": "Grader Feedback"
                }
        score_msg = {
                'correct': True,
                'score': 4,
                'msg': 'Grader Message',
                'feedback': json.dumps(feedback),
                'grader_type': 'IN',
                'grader_id': '1',
                'submission_id': '1',
                'success': True,
                'rubric_scores': [0],
                'rubric_scores_complete': True,
                'rubric_xml': etree.tostring(self.rubric)
                }
        get = {'queuekey': "abcd",
                'xqueue_body': json.dumps(score_msg)}
        self.openendedmodule.update_score(get, test_system)

    def test_latest_post_assessment(self):
        self.update_score_single()
        assessment = self.openendedmodule.latest_post_assessment(test_system)
        self.assertFalse(assessment == '')
        # check for errors
        self.assertFalse('errors' in assessment)

    def test_update_score(self):
        self.update_score_single()
        score = self.openendedmodule.latest_score()
        self.assertEqual(score, 4)

class CombinedOpenEndedModuleTest(unittest.TestCase):
    location = Location(["i4x", "edX", "open_ended", "combinedopenended",
                         "SampleQuestion"])

    prompt = "<prompt>This is a question prompt</prompt>"
    rubric = '''<rubric><rubric>
        <category>
        <description>Response Quality</description>
        <option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
        </category>
         </rubric></rubric>'''
    max_score = 3

    metadata = {'attempts': '10', 'max_score': max_score}

    static_data = {
            'max_attempts': 20,
            'prompt': prompt,
            'rubric': rubric,
            'max_score': max_score,
            'display_name': 'Name',
            'accept_file_upload' : False,
            'rewrite_content_links' : "",
            }

    oeparam = etree.XML('''
      <openendedparam>
            <initial_display>Enter essay here.</initial_display>
            <answer_display>This is the answer.</answer_display>
            <grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
        </openendedparam>
    ''')

    task_xml1 = '''
                <selfassessment>
                    <hintprompt>
                        What hint about this problem would you give to someone?
                    </hintprompt>
                    <submitmessage>
                        Save Succcesful.  Thanks for participating!
                    </submitmessage>
                </selfassessment>
            '''
    task_xml2 = '''
    <openended min_score_to_attempt="1" max_score_to_attempt="1">
		        <openendedparam>
		      		<initial_display>Enter essay here.</initial_display>
		      		<answer_display>This is the answer.</answer_display>
		      		<grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
		    	</openendedparam>
		</openended>'''
    definition = {'prompt': etree.XML(prompt), 'rubric': etree.XML(rubric), 'task_xml': [task_xml1, task_xml2]}
    descriptor = Mock()

    def setUp(self):
        self.combinedoe = CombinedOpenEndedV1Module(test_system, self.location, self.definition, self.descriptor, static_data = self.static_data, metadata=self.metadata)

    def test_get_tag_name(self):
        name = self.combinedoe.get_tag_name("<t>Tag</t>")
        self.assertEqual(name, "t")

    def test_get_last_response(self):
        response_dict = self.combinedoe.get_last_response(0)
        self.assertEqual(response_dict['type'], "selfassessment")
        self.assertEqual(response_dict['max_score'], self.max_score)
        self.assertEqual(response_dict['state'], CombinedOpenEndedV1Module.INITIAL)

    def test_update_task_states(self):
        changed = self.combinedoe.update_task_states()
        self.assertFalse(changed)

        current_task = self.combinedoe.current_task
        current_task.change_state(CombinedOpenEndedV1Module.DONE)
        changed = self.combinedoe.update_task_states()

        self.assertTrue(changed)
<|endoftext|>


<commit_before>import os
from datetime import *
from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import DocType, Date, Nested, Boolean, MetaField
from elasticsearch_dsl import analyzer, InnerObjectWrapper, Text, Integer
from elasticsearch import serializer, compat, exceptions
from elasticsearch_dsl import Search
from elasticsearch_dsl import Q
from elasticsearch_dsl import Index
import re
try:
    import simplejson as json
except ImportError:
    import json

class JSONSerializerPython2(serializer.JSONSerializer):
    """Override elasticsearch library serializer to ensure it encodes utf characters during json dump.
    See original at: https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/serializer.py#L42
    A description of how ensure_ascii encodes unicode characters to ensure they can be sent across the wire
    as ascii can be found here: https://docs.python.org/2/library/json.html#basic-usage
    """
    def dumps(self, data):
        # don't serialize strings
        if isinstance(data, compat.string_types):
            return data
        try:
            return json.dumps(data, default=self.default, ensure_ascii=True)
        except (ValueError, TypeError) as e:
            raise exceptions.SerializationError(data, e)

def elasticsearch_pages(context, sort, page):
    result_limit = int(os.environ['RESULT_LIMIT'])
    max_result_limit = int(os.environ['MAX_RESULT_LIMIT'])
    start = (page - 1) * result_limit
    end   = start + result_limit
    domain_query = Q("term", is_banned=False)
    if context["is_up"]:
        domain_query = domain_query & Q("term", is_up=True)
    if not context["show_fh_default"]:
        domain_query = domain_query & Q("term", is_crap=False)
    if not context["show_subdomains"]:
        domain_query = domain_query & Q("term", is_subdomain=False)
    if context["rep"] == "genuine":
        domain_query = domain_query & Q("term", is_genuine=True)
    if context["rep"] == "fake":
        domain_query = domain_query & Q("term", is_fake=True)

    limit = max_result_limit if context["more"] else result_limit

    has_parent_query = Q("has_parent", type="domain", query=domain_query)
    query = Search().filter(has_parent_query).query(Q("match", body_stripped=context['search']))
    query = query.highlight_options(order='score', encoder='html').highlight('body_stripped')[start:end]
    query = query.source(['title','domain_id','created_at', 'visited_at']).params(request_cache=True)
    return query.execute()

def is_elasticsearch_enabled():
    return ('ELASTICSEARCH_ENABLED' in os.environ and os.environ['ELASTICSEARCH_ENABLED'].lower()=='true')

class DomainDocType(DocType):
    title = Text(analyzer="snowball")
    created_at = Date()
    visited_at = Date()
    last_alive = Date()
    is_up      = Boolean()
    is_fake    = Boolean()
    is_genuine = Boolean()
    is_crap    = Boolean()
    is_banned  = Boolean()
    url        = Text()
    is_subdomain = Boolean()
    ssl        = Boolean()
    port       = Integer()

    class Meta:
        name = 'domain'
        doc_type = 'domain'

    @classmethod
    def get_indexable(cls):
        return cls.get_model().get_objects()

    @classmethod
    def from_obj(klass, obj):
        return klass(
            meta={'id': obj.host},
            title=obj.title,
            created_at=obj.created_at,
            visited_at=obj.visited_at,
            is_up=obj.is_up,
            is_fake=obj.is_fake,
            is_genuine=obj.is_fake,
            is_crap=obj.is_crap,
            is_banned=obj.is_banned,
            url=obj.index_url(),
            is_subdomain=obj.is_subdomain,
            ssl=obj.ssl,
            port=obj.port
        )

    @classmethod
    def set_isup(klass, obj, is_up):
    	dom = klass(meta={'id': obj.host})
    	dom.update(is_up=is_up)

class PageDocType(DocType):
    html_strip = analyzer('html_strip',
        tokenizer="standard",
        filter=["standard", "lowercase", "stop", "snowball", "asciifolding"],
        char_filter=["html_strip"]
    )

    title = Text(analyzer="snowball")
    created_at   =  Date()
    visited_at    = Date()
    code          = Integer()
    body          = Text()
    domain_id     = Integer()
    body_stripped = Text(analyzer=html_strip, term_vector="with_positions_offsets")
    is_frontpage  = Boolean()
    nid           = Integer()

    class Meta:
        name = 'page'
        doc_type = 'page'
        parent = MetaField(type='domain')

    @classmethod
    def get_indexable(cls):
        return cls.get_model().get_objects()

    @classmethod
    def from_obj(klass, obj, body):
        return klass(
            meta={'id':obj.url, 'routing':obj.domain.host, 'parent':obj.domain.host},
            title=obj.title,
            created_at=obj.created_at,
            visited_at=obj.visited_at,
            is_frontpage=obj.is_frontpage,
            code=obj.code,
            domain_id=obj.domain.id,
            body=body,
            body_stripped=re.sub('<[^<]+?>', '', body),
            nid=obj.id
        )

hidden_services = None

if is_elasticsearch_enabled():
    connections.create_connection(hosts=[os.environ['ELASTICSEARCH_HOST']], serializer=JSONSerializerPython2(), timeout=int(os.environ['ELASTICSEARCH_TIMEOUT']))
    hidden_services = Index('hiddenservices')
    hidden_services.doc_type(DomainDocType)
    hidden_services.doc_type(PageDocType)

def migrate():
    hidden_services = Index('hiddenservices')
    hidden_services.delete(ignore=404)
    hidden_services = Index('hiddenservices')
    hidden_services.doc_type(DomainDocType)
    hidden_services.doc_type(PageDocType)
    hidden_services.settings(number_of_shards=8, number_of_replicas=1)
    hidden_services.create()

<commit_msg>fix fake/genuine insert<commit_after>import os
from datetime import *
from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import DocType, Date, Nested, Boolean, MetaField
from elasticsearch_dsl import analyzer, InnerObjectWrapper, Text, Integer
from elasticsearch import serializer, compat, exceptions
from elasticsearch_dsl import Search
from elasticsearch_dsl import Q
from elasticsearch_dsl import Index
import re
try:
    import simplejson as json
except ImportError:
    import json

class JSONSerializerPython2(serializer.JSONSerializer):
    """Override elasticsearch library serializer to ensure it encodes utf characters during json dump.
    See original at: https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/serializer.py#L42
    A description of how ensure_ascii encodes unicode characters to ensure they can be sent across the wire
    as ascii can be found here: https://docs.python.org/2/library/json.html#basic-usage
    """
    def dumps(self, data):
        # don't serialize strings
        if isinstance(data, compat.string_types):
            return data
        try:
            return json.dumps(data, default=self.default, ensure_ascii=True)
        except (ValueError, TypeError) as e:
            raise exceptions.SerializationError(data, e)

def elasticsearch_pages(context, sort, page):
    result_limit = int(os.environ['RESULT_LIMIT'])
    max_result_limit = int(os.environ['MAX_RESULT_LIMIT'])
    start = (page - 1) * result_limit
    end   = start + result_limit
    domain_query = Q("term", is_banned=False)
    if context["is_up"]:
        domain_query = domain_query & Q("term", is_up=True)
    if not context["show_fh_default"]:
        domain_query = domain_query & Q("term", is_crap=False)
    if not context["show_subdomains"]:
        domain_query = domain_query & Q("term", is_subdomain=False)
    if context["rep"] == "genuine":
        domain_query = domain_query & Q("term", is_genuine=True)
    if context["rep"] == "fake":
        domain_query = domain_query & Q("term", is_fake=True)

    limit = max_result_limit if context["more"] else result_limit

    has_parent_query = Q("has_parent", type="domain", query=domain_query)
    query = Search().filter(has_parent_query).query(Q("match", body_stripped=context['search']))
    query = query.highlight_options(order='score', encoder='html').highlight('body_stripped')[start:end]
    query = query.source(['title','domain_id','created_at', 'visited_at']).params(request_cache=True)
    return query.execute()

def is_elasticsearch_enabled():
    return ('ELASTICSEARCH_ENABLED' in os.environ and os.environ['ELASTICSEARCH_ENABLED'].lower()=='true')

class DomainDocType(DocType):
    title = Text(analyzer="snowball")
    created_at = Date()
    visited_at = Date()
    last_alive = Date()
    is_up      = Boolean()
    is_fake    = Boolean()
    is_genuine = Boolean()
    is_crap    = Boolean()
    is_banned  = Boolean()
    url        = Text()
    is_subdomain = Boolean()
    ssl        = Boolean()
    port       = Integer()

    class Meta:
        name = 'domain'
        doc_type = 'domain'

    @classmethod
    def get_indexable(cls):
        return cls.get_model().get_objects()

    @classmethod
    def from_obj(klass, obj):
        return klass(
            meta={'id': obj.host},
            title=obj.title,
            created_at=obj.created_at,
            visited_at=obj.visited_at,
            is_up=obj.is_up,
            is_fake=obj.is_fake,
            is_genuine=obj.is_genuine,
            is_crap=obj.is_crap,
            is_banned=obj.is_banned,
            url=obj.index_url(),
            is_subdomain=obj.is_subdomain,
            ssl=obj.ssl,
            port=obj.port
        )

    @classmethod
    def set_isup(klass, obj, is_up):
    	dom = klass(meta={'id': obj.host})
    	dom.update(is_up=is_up)

class PageDocType(DocType):
    html_strip = analyzer('html_strip',
        tokenizer="standard",
        filter=["standard", "lowercase", "stop", "snowball", "asciifolding"],
        char_filter=["html_strip"]
    )

    title = Text(analyzer="snowball")
    created_at   =  Date()
    visited_at    = Date()
    code          = Integer()
    body          = Text()
    domain_id     = Integer()
    body_stripped = Text(analyzer=html_strip, term_vector="with_positions_offsets")
    is_frontpage  = Boolean()
    nid           = Integer()

    class Meta:
        name = 'page'
        doc_type = 'page'
        parent = MetaField(type='domain')

    @classmethod
    def get_indexable(cls):
        return cls.get_model().get_objects()

    @classmethod
    def from_obj(klass, obj, body):
        return klass(
            meta={'id':obj.url, 'routing':obj.domain.host, 'parent':obj.domain.host},
            title=obj.title,
            created_at=obj.created_at,
            visited_at=obj.visited_at,
            is_frontpage=obj.is_frontpage,
            code=obj.code,
            domain_id=obj.domain.id,
            body=body,
            body_stripped=re.sub('<[^<]+?>', '', body),
            nid=obj.id
        )

hidden_services = None

if is_elasticsearch_enabled():
    connections.create_connection(hosts=[os.environ['ELASTICSEARCH_HOST']], serializer=JSONSerializerPython2(), timeout=int(os.environ['ELASTICSEARCH_TIMEOUT']))
    hidden_services = Index('hiddenservices')
    hidden_services.doc_type(DomainDocType)
    hidden_services.doc_type(PageDocType)

def migrate():
    hidden_services = Index('hiddenservices')
    hidden_services.delete(ignore=404)
    hidden_services = Index('hiddenservices')
    hidden_services.doc_type(DomainDocType)
    hidden_services.doc_type(PageDocType)
    hidden_services.settings(number_of_shards=8, number_of_replicas=1)
    hidden_services.create()

<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs fio benchmarks.

Man: http://manpages.ubuntu.com/manpages/natty/man1/fio.1.html
Quick howto: http://www.bluestop.org/fio/HOWTO.txt
"""

import json
import logging
import posixpath
import re

import jinja2

from perfkitbenchmarker import flags
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.packages import fio

LOCAL_JOB_FILE_NAME = 'fio.job'  # used with vm_util.PrependTempDir()
REMOTE_JOB_FILE_PATH = posixpath.join(vm_util.VM_TMP_DIR, 'fio.job')
DEFAULT_TEMP_FILE_NAME = 'fio-temp-file'
MAX_FILE_SIZE_GB = 100
DISK_USABLE_SPACE_FRACTION = 0.9

FLAGS = flags.FLAGS

flags.DEFINE_string('fio_jobfile', None,
                    'Job file that fio will use. Giving a custom job file '
                    'overrides the other fio options.')
flags.DEFINE_boolean('against_device', False,
                     'If true, test directly against the scratch disk.')
flags.DEFINE_string('device_fill_size', '100%',
                    'The amount of device to fill in prepare stage. '
                    'This flag is only valid when against_device=True. '
                    'The valid value can either be an integer, which '
                    'represents the number of bytes to fill or a '
                    'percentage, which represents the percentage '
                    'of the device.')
flags.DEFINE_string('io_depths', '1',
                    'IO queue depths to run on. Can specify a single number, '
                    'like --io_depths=1, or a range, like --io_depths=1-4')

FLAGS_IGNORED_FOR_CUSTOM_JOBFILE = {
    'against_device', 'device_fill_size', 'io_depth'}

IODEPTHS_REGEXP = re.compile(r'(\d+)(-(\d+))?')

def IODepthsValidator(string):
  match = IODEPTHS_REGEXP.match(string)
  return match and match.end() == len(string) and int(match.group(1)) > 0

flags.RegisterValidator('io_depths',
                        IODepthsValidator,
                        message='--io_depths must be an integer '
                                'or a range of integers, all > 0')

BENCHMARK_INFO = {'name': 'fio',
                  'description': 'Runs fio in sequential, random, read '
                                 'and write modes.',
                  'scratch_disk': True,
                  'num_machines': 1}

JOB_FILE_TEMPLATE = """
[global]
ioengine=libaio
invalidate=1
blocksize=4k
direct=1
runtime=10m
time_based
filename={{filename}}
do_verify=0
verify_fatal=0
randrepeat=0
size={{size}}

{% for rwkind in ('randread', 'randwrite') %}
{% for iodepth in iodepths %}
[{{rwkind}}-io-depth-{{iodepth}}]
stonewall
rw={{rwkind}}
iodepth={{iodepth}}

{% endfor %}
{% endfor %}
"""

def GetIODepths(io_depths):
  """Parse the io_depths parameter.

  Args:
    io_depths: a string in the format of the --io_depths flag.

  Returns:
    An iterable of integers.

  Raises:
    ValueError if FLAGS.io_depths doesn't follow a format it recognizes.
  """

  match = IODEPTHS_REGEXP.match(io_depths)

  if match.group(2) is None:
    return [int(match.group(1))]
  else:
    return range(int(match.group(1)), int(match.group(3)) + 1)

def GenerateJobFileString(mount_point, against_device,
                          device_fill_size, io_depths, device_size):
  """Write a fio job file.

  Args:
    mount_point: the mount point of the disk we're testing against.
    against_device: bool. True if we're using a raw disk.
    device_fill_size: string. The amount of the disk to pre-fill.
    io_depths: string. The contents of the io_depths flag.
    device_size: int. The size of the device, in gigabytes.

  Returns:
    The contents of a fio job file, as a string.
  """

  if against_device:
    filename = mount_point
    size = device_fill_size
  else:
    filename = posixpath.join(mount_point, DEFAULT_TEMP_FILE_NAME)
    size = str(min(MAX_FILE_SIZE_GB,
                   int(DISK_USABLE_SPACE_FRACTION * device_size))) + 'G'

  return str(jinja2.Template(JOB_FILE_TEMPLATE,
                             undefined=jinja2.StrictUndefined).render(
      filename=filename,
      size=size,
      iodepths=GetIODepths(io_depths)))

def JobFileString(fio_jobfile, mount_point, against_device,
                  device_fill_size, io_depths, device_size):
  """Get the contents of our job file.

  Args:
    fio_jobfile: string or False. The path to the user's jobfile, if provided.
    mount_point: the mount point of the disk we're testing against.
    against_device: bool. True if we're using a raw disk.
    device_fill_size: string. The amount of the disk to pre-fill.
    io_depths: string. The contents of the io_depths flag.
    device_size: int. The size of the device, in gigabytes.

    vm: the virtual_machine.BaseVirtualMachine that we will run on.

  Returns:
    A string containing the user's job file.
  """

  if fio_jobfile:
    with open(fio_jobfile, 'r') as jobfile:
      return jobfile.read()
  else:
    return GenerateJobFileString(mount_point, against_device,
                                 device_fill_size, io_depths, device_size)

def GetInfo():
  return BENCHMARK_INFO

def Prepare(benchmark_spec):
  """Prepare the virtual machine to run FIO.

     This includes installing fio, bc, and libaio1 and insuring that
     the attached disk is large enough to support the fio
     benchmark. We also make sure the job file is always located at
     the same path on the local machine.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  """

  if FLAGS.fio_jobfile:
    ignored_flags = {flag_name for flag_name in FLAGS_IGNORED_FOR_CUSTOM_JOBFILE
                     if FLAGS[flag_name].present}

    if ignored_flags:
      logging.warning('Fio job file specified. Ignoring options "%s"',
                      ', '.join(ignored_flags))
  if FLAGS['device_fill_size'].present and not FLAGS['against_device'].present:
    logging.warning('--device_fill_size has no effect without --against_device')

  vm = benchmark_spec.vms[0]
  logging.info('FIO prepare on %s', vm)
  vm.Install('fio')

  disk = vm.scratch_disks[0]
  if FLAGS.against_device and not FLAGS.fio_jobfile:
    mount_point = disk.mount_point
    logging.info('Umount scratch disk on %s at %s', vm, mount_point)
    vm.RemoteCommand('sudo umount %s' % mount_point)

    device_path = disk.GetDevicePath()
    logging.info('Fill scratch disk on %s at %s', vm, device_path)
    command = (
        ('sudo %s --filename=%s --ioengine=libaio '
         '--name=fill-device --blocksize=512k --iodepth=64 '
         '--rw=write --direct=1 --size=%s') %
        (fio.FIO_PATH, device_path, FLAGS.device_fill_size))
    vm.RemoteCommand(command)

  job_file_path = vm_util.PrependTempDir(LOCAL_JOB_FILE_NAME)
  with open(job_file_path, 'w') as job_file:
    job_file.write(JobFileString(FLAGS.fio_jobfile,
                                 disk.mount_point,
                                 FLAGS.against_device,
                                 FLAGS.device_fill_size,
                                 FLAGS.io_depths,
                                 disk.disk_size))
    logging.info('Wrote fio job file at %s', job_file_path)

  vm.PushFile(job_file_path, REMOTE_JOB_FILE_PATH)

def Run(benchmark_spec):
  """Spawn fio and gather the results.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  vm = benchmark_spec.vms[0]
  logging.info('FIO running on %s', vm)

  fio_command = 'sudo %s --output-format=json %s' % (fio.FIO_PATH,
                                                     REMOTE_JOB_FILE_PATH)
  # TODO(user): This only gives results at the end of a job run
  #      so the program pauses here with no feedback to the user.
  #      This is a pretty lousy experience.
  logging.info('FIO Results:')
  stdout, stderr = vm.RemoteCommand(fio_command, should_log=True)

  disk = vm.scratch_disks[0]
  return fio.ParseResults(JobFileString(FLAGS.fio_jobfile,
                                        disk.mount_point,
                                        FLAGS.against_device,
                                        FLAGS.device_fill_size,
                                        FLAGS.io_depths,
                                        disk.disk_size),
                          json.loads(stdout))

def Cleanup(benchmark_spec):
  """Uninstall packages required for fio and remove benchmark files.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vm = benchmark_spec.vms[0]
  logging.info('FIO Cleanup up on %s', vm)
  vm.RemoveFile(REMOTE_JOB_FILE_PATH)
  if not FLAGS.against_device and not FLAGS.fio_jobfile:
    # If the user supplies their own job file, then they have to clean
    # up after themselves, because we don't know their temp file name.
    vm.RemoveFile(posixpath.join(vm.GetScratchDir(), DEFAULT_TEMP_FILE_NAME))
<commit_msg>Respond to PR comments<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs fio benchmarks.

Man: http://manpages.ubuntu.com/manpages/natty/man1/fio.1.html
Quick howto: http://www.bluestop.org/fio/HOWTO.txt
"""

import json
import logging
import posixpath
import re

import jinja2

from perfkitbenchmarker import flags
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.packages import fio

LOCAL_JOB_FILE_NAME = 'fio.job'  # used with vm_util.PrependTempDir()
REMOTE_JOB_FILE_PATH = posixpath.join(vm_util.VM_TMP_DIR, 'fio.job')
DEFAULT_TEMP_FILE_NAME = 'fio-temp-file'
MAX_FILE_SIZE_GB = 100
DISK_USABLE_SPACE_FRACTION = 0.9

FLAGS = flags.FLAGS

flags.DEFINE_string('fio_jobfile', None,
                    'Job file that fio will use. Giving a custom job file '
                    'overrides the other fio options.')
flags.DEFINE_boolean('against_device', False,
                     'If true, test directly against the scratch disk.')
flags.DEFINE_string('device_fill_size', '100%',
                    'The amount of device to fill in prepare stage. '
                    'This flag is only valid when against_device=True. '
                    'The valid value can either be an integer, which '
                    'represents the number of bytes to fill or a '
                    'percentage, which represents the percentage '
                    'of the device.')
flags.DEFINE_string('io_depths', '1',
                    'IO queue depths to run on. Can specify a single number, '
                    'like --io_depths=1, or a range, like --io_depths=1-4')

FLAGS_IGNORED_FOR_CUSTOM_JOBFILE = {
    'against_device', 'device_fill_size', 'io_depth'}

IODEPTHS_REGEXP = re.compile(r'(\d+)(-(\d+))?')

def IODepthsValidator(string):
  match = IODEPTHS_REGEXP.match(string)
  return match and match.end() == len(string) and int(match.group(1)) > 0

flags.RegisterValidator('io_depths',
                        IODepthsValidator,
                        message='--io_depths must be an integer '
                                'or a range of integers, all > 0')

BENCHMARK_INFO = {'name': 'fio',
                  'description': 'Runs fio in sequential, random, read '
                                 'and write modes.',
                  'scratch_disk': True,
                  'num_machines': 1}

JOB_FILE_TEMPLATE = """
[global]
ioengine=libaio
invalidate=1
blocksize=4k
direct=1
runtime=10m
time_based
filename={{filename}}
do_verify=0
verify_fatal=0
randrepeat=0
size={{size}}

{% for rwkind in ('randread', 'randwrite') %}
{% for iodepth in iodepths %}
[{{rwkind}}-io-depth-{{iodepth}}]
stonewall
rw={{rwkind}}
iodepth={{iodepth}}

{% endfor %}
{% endfor %}
"""

def GetIODepths(io_depths):
  """Parse the io_depths parameter.

  Args:
    io_depths: a string in the format of the --io_depths flag.

  Returns:
    An iterable of integers.

  Raises:
    ValueError if FLAGS.io_depths doesn't follow a format it recognizes.
  """

  match = IODEPTHS_REGEXP.match(io_depths)

  if match.group(2) is None:
    return [int(match.group(1))]
  else:
    return range(int(match.group(1)), int(match.group(3)) + 1)

def GenerateJobFileString(mount_point, against_device,
                          device_fill_size, io_depths, device_size):
  """Write a fio job file.

  Args:
    mount_point: the mount point of the disk we're testing against.
    against_device: bool. True if we're using a raw disk.
    device_fill_size: string. The amount of the disk to pre-fill.
    io_depths: iterable. The IO queue depths to test.
    device_size: int. The size of the device, in gigabytes.

  Returns:
    The contents of a fio job file, as a string.
  """

  if against_device:
    filename = mount_point
    size = device_fill_size
  else:
    filename = posixpath.join(mount_point, DEFAULT_TEMP_FILE_NAME)
    size = str(min(MAX_FILE_SIZE_GB,
                   int(DISK_USABLE_SPACE_FRACTION * device_size))) + 'G'

  return str(jinja2.Template(JOB_FILE_TEMPLATE,
                             undefined=jinja2.StrictUndefined).render(
      filename=filename,
      size=size,
      iodepths=io_depths))

def JobFileString(fio_jobfile, mount_point, against_device,
                  device_fill_size, io_depths, device_size):
  """Get the contents of our job file.

  Args:
    fio_jobfile: string or False. The path to the user's jobfile, if provided.
    mount_point: the mount point of the disk we're testing against.
    against_device: bool. True if we're using a raw disk.
    device_fill_size: string. The amount of the disk to pre-fill.
    io_depths: iterable. The IO queue depths to test.
    device_size: int. The size of the device, in gigabytes.

    vm: the virtual_machine.BaseVirtualMachine that we will run on.

  Returns:
    A string containing the user's job file.
  """

  if fio_jobfile:
    with open(fio_jobfile, 'r') as jobfile:
      return jobfile.read()
  else:
    return GenerateJobFileString(mount_point, against_device,
                                 device_fill_size, io_depths, device_size)

def GetInfo():
  return BENCHMARK_INFO

def Prepare(benchmark_spec):
  """Prepare the virtual machine to run FIO.

     This includes installing fio, bc, and libaio1 and insuring that
     the attached disk is large enough to support the fio
     benchmark. We also make sure the job file is always located at
     the same path on the local machine.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  """

  if FLAGS.fio_jobfile:
    ignored_flags = {'--' + flag_name
                     for flag_name in FLAGS_IGNORED_FOR_CUSTOM_JOBFILE
                     if FLAGS[flag_name].present}

    if ignored_flags:
      logging.warning('Fio job file specified. Ignoring options "%s"',
                      ', '.join(ignored_flags))
  if FLAGS['device_fill_size'].present and not FLAGS['against_device'].present:
    logging.warning('--device_fill_size has no effect without --against_device')

  vm = benchmark_spec.vms[0]
  logging.info('FIO prepare on %s', vm)
  vm.Install('fio')

  disk = vm.scratch_disks[0]
  if FLAGS.against_device and not FLAGS.fio_jobfile:
    mount_point = disk.mount_point
    logging.info('Umount scratch disk on %s at %s', vm, mount_point)
    vm.RemoteCommand('sudo umount %s' % mount_point)

    device_path = disk.GetDevicePath()
    logging.info('Fill scratch disk on %s at %s', vm, device_path)
    command = (
        ('sudo %s --filename=%s --ioengine=libaio '
         '--name=fill-device --blocksize=512k --iodepth=64 '
         '--rw=write --direct=1 --size=%s') %
        (fio.FIO_PATH, device_path, FLAGS.device_fill_size))
    vm.RemoteCommand(command)

  job_file_path = vm_util.PrependTempDir(LOCAL_JOB_FILE_NAME)
  with open(job_file_path, 'w') as job_file:
    job_file.write(JobFileString(FLAGS.fio_jobfile,
                                 disk.mount_point,
                                 FLAGS.against_device,
                                 FLAGS.device_fill_size,
                                 GetIODepths(FLAGS.io_depths),
                                 disk.disk_size))
    logging.info('Wrote fio job file at %s', job_file_path)

  vm.PushFile(job_file_path, REMOTE_JOB_FILE_PATH)

def Run(benchmark_spec):
  """Spawn fio and gather the results.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of samples in the form of 3 or 4 tuples. The tuples contain
        the sample metric (string), value (float), and unit (string).
        If a 4th element is included, it is a dictionary of sample
        metadata.
  """
  vm = benchmark_spec.vms[0]
  logging.info('FIO running on %s', vm)

  fio_command = 'sudo %s --output-format=json %s' % (fio.FIO_PATH,
                                                     REMOTE_JOB_FILE_PATH)
  # TODO(user): This only gives results at the end of a job run
  #      so the program pauses here with no feedback to the user.
  #      This is a pretty lousy experience.
  logging.info('FIO Results:')
  stdout, stderr = vm.RemoteCommand(fio_command, should_log=True)

  disk = vm.scratch_disks[0]
  return fio.ParseResults(JobFileString(FLAGS.fio_jobfile,
                                        disk.mount_point,
                                        FLAGS.against_device,
                                        FLAGS.device_fill_size,
                                        GetIODepths(FLAGS.io_depths),
                                        disk.disk_size),
                          json.loads(stdout))

def Cleanup(benchmark_spec):
  """Uninstall packages required for fio and remove benchmark files.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vm = benchmark_spec.vms[0]
  logging.info('FIO Cleanup up on %s', vm)
  vm.RemoveFile(REMOTE_JOB_FILE_PATH)
  if not FLAGS.against_device and not FLAGS.fio_jobfile:
    # If the user supplies their own job file, then they have to clean
    # up after themselves, because we don't know their temp file name.
    vm.RemoveFile(posixpath.join(vm.GetScratchDir(), DEFAULT_TEMP_FILE_NAME))
<|endoftext|>


<commit_before>import unittest

from calpack import models

class Test_Repr(unittest.TestCase):

    def test_simple_repr(self):
        class two_int_pkt(models.Packet):
            field1 = models.IntField()
            field2 = models.IntField()

        pkt = two_int_pkt()

        expected = "two_int_pkt(field1=0, field2=0)"
        self.assertEqual(repr(pkt), expected)

        pkt.field1 = 1234
        pkt.field2 = 5678

        expected = "two_int_pkt(field1=1234, field2=5678)"
        self.assertEqual(repr(pkt), expected)
<commit_msg>support for Python 2<commit_after>import unittest

from calpack import models
from calpack.utils import PY2

class Test_Repr(unittest.TestCase):

    def test_simple_repr(self):
        class two_int_pkt(models.Packet):
            field1 = models.IntField()
            field2 = models.IntField()

        pkt = two_int_pkt()

        expected_py3 = "two_int_pkt(field1=0, field2=0)"
        expected_py2 = "two_int_pkt(field1=0L, field2=0L)"
        self.assertEqual(repr(pkt), expected_py2 if PY2 else expected_py3)

        pkt.field1 = 1234
        pkt.field2 = 5678

        expected_py3 = "two_int_pkt(field1=1234, field2=5678)"
        expected_py2 = "two_int_pkt(field1=1234, field2=5678)"
        self.assertEqual(repr(pkt), expected_py2 if PY2 else expected_py3)
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from .utils.dataIO import dataIO
from .utils.dataIO import fileIO
from .utils import checks
import time
import os
import csv
import requests

class ClanMod:

    def __init__(self, bot):
        self.bot = bot

    @commands.command(no_pm=True, pass_context=True)
    @checks.admin_or_permissions(manage_nicknames=True)
    async def assign_clan(self, ctx, user : discord.Member, *, clanname=""):
        """Change user's nickname to match his clan

        Leaving the nickname empty will remove it."""
        nickname = '[{}] {}'.format(clanname.strip(), user.name)
        if clanname == "":
            nickname = None
        try:
            await self.bot.change_nickname(user, nickname)
            await self.bot.say("Done.")
        except discord.Forbidden:
            await self.bot.say("I cannot do that, I lack the "
                "\"Manage Nicknames\" permission.")

    @commands.command(pass_context=True, no_pm=True)
    async def profile(self,ctx, *, user : discord.Member=None):
        """Displays a user profile."""
        if user == None:
            user = ctx.message.author
        channel = ctx.message.channel
        server = user.server
        curr_time = time.time()
        # creates user if doesn't exist
        await self._create_user(user, server)
        userinfo = fileIO("data/clanmod/users/{}/info.json".format(user.id), "load")

        await self.bot.say('Temporary User Profile placeholder statement for user {}'.format(user))

    async def on_attachment(self, message):
        if len(message.attachments):
            for attachment in message.attachments:
                if attachment['filename'] == 'champions.csv':
                    self._parse_champions_csv(self, message)

    # handles user creation, adding new server, blocking
    async def _create_user(self, user, server):
        await self.bot.say('DEBUG: _create_user has been initialized')
        try:
            if not os.path.exists("data/clanmod/users/{}".format(user.id)):
                os.makedirs("data/clanmod/users/{}".format(user.id))
                new_account = {
                    "servers": {},
                    "created": time.time(),
                    "last_updated": time.time()
                }
                fileIO("data/clanmod/users/{}/info.json".format(user.id), "save", new_account)

            userinfo = fileIO("data/clanmod/users/{}/info.json".format(user.id), "load")
            if server.id not in userinfo["servers"]:
                userinfo["servers"][server.id] = {
                    "clan":{},
                    "battlegroup":{},
                    "champs": {},
                    "prestige": 0,
                    "top5": {},
                    "aq": {},
                    "awd": {},
                    "awo": {},
                    "max5":{},
                    "last_updated": time.time()
                }
                fileIO("data/clanmod/users/{}/info.json".format(user.id), "save", userinfo)
        except AttributeError as e:
            pass

    async def _parse_champions_csv(self, message):

        user = message.author
        server = user.server
        channel = message.channel
        await self.bot.send_message(channel, 'DEBUG: Message attachment detected')
        for attachment in message.attachments:
            await self._create_user(user,server)
            await self.bot.send_message(channel, 'attachment[0]: {}'.format(attachment))
            url = open(attachment[0]['url'])
            with requests.Session() as s:
                download = s.get(url)
                decoded_content = download.content.decode('utf-8')
                cr = csv.reader(decoded_content.splitlines(),delimiter=',')
                for i in 10:
                    temp = cr[i]
            await self.bot.say(temp)
            await self.bot.say('DEBUG: CSV file opened')

#-------------- setup -------------
def check_folders():
    if not os.path.exists("data/clanmod"):
        print("Creating data/clanmod folder...")
        os.makedirs("data/clanmod")

    if not os.path.exists("data/clanmod/users"):
        print("Creating data/clanmod/users folder...")
        os.makedirs("data/clanmod/users")
        transfer_info()

def setup(bot):
    check_folders()
    n = ClanMod(bot)
    bot.add_cog(n)
    bot.add_listener(n.on_attachment, name='on_message')
<commit_msg>on_message debug<commit_after>import discord
from discord.ext import commands
from .utils.dataIO import dataIO
from .utils.dataIO import fileIO
from .utils import checks
import time
import os
import csv
import requests

class ClanMod:

    def __init__(self, bot):
        self.bot = bot

    @commands.command(no_pm=True, pass_context=True)
    @checks.admin_or_permissions(manage_nicknames=True)
    async def assign_clan(self, ctx, user : discord.Member, *, clanname=""):
        """Change user's nickname to match his clan

        Leaving the nickname empty will remove it."""
        nickname = '[{}] {}'.format(clanname.strip(), user.name)
        if clanname == "":
            nickname = None
        try:
            await self.bot.change_nickname(user, nickname)
            await self.bot.say("Done.")
        except discord.Forbidden:
            await self.bot.say("I cannot do that, I lack the "
                "\"Manage Nicknames\" permission.")

    @commands.command(pass_context=True, no_pm=True)
    async def profile(self,ctx, *, user : discord.Member=None):
        """Displays a user profile."""
        if user == None:
            user = ctx.message.author
        channel = ctx.message.channel
        server = user.server
        curr_time = time.time()
        # creates user if doesn't exist
        await self._create_user(user, server)
        userinfo = fileIO("data/clanmod/users/{}/info.json".format(user.id), "load")

        await self.bot.say('Temporary User Profile placeholder statement for user {}'.format(user))

    async def on_attachment(self, message):
        await self.bot.say('DEBUG: on_attachment initialized')
        if len(message.attachments) > 0:
            for attachment in message.attachments:
                if attachment['filename'] == 'champions.csv':
                    self._parse_champions_csv(self, message)

    # handles user creation, adding new server, blocking
    async def _create_user(self, user, server):
        await self.bot.say('DEBUG: _create_user has been initialized')
        try:
            if not os.path.exists("data/clanmod/users/{}".format(user.id)):
                os.makedirs("data/clanmod/users/{}".format(user.id))
                new_account = {
                    "servers": {},
                    "created": time.time(),
                    "last_updated": time.time()
                }
                fileIO("data/clanmod/users/{}/info.json".format(user.id), "save", new_account)

            userinfo = fileIO("data/clanmod/users/{}/info.json".format(user.id), "load")
            if server.id not in userinfo["servers"]:
                userinfo["servers"][server.id] = {
                    "clan":{},
                    "battlegroup":{},
                    "champs": {},
                    "prestige": 0,
                    "top5": {},
                    "aq": {},
                    "awd": {},
                    "awo": {},
                    "max5":{},
                    "last_updated": time.time()
                }
                fileIO("data/clanmod/users/{}/info.json".format(user.id), "save", userinfo)
        except AttributeError as e:
            pass

    async def _parse_champions_csv(self, message):

        user = message.author
        server = user.server
        channel = message.channel
        await self.bot.send_message(channel, 'DEBUG: Message attachment detected')
        for attachment in message.attachments:
            await self._create_user(user,server)
            await self.bot.send_message(channel, 'attachment[0]: {}'.format(attachment))
            url = open(attachment[0]['url'])
            with requests.Session() as s:
                download = s.get(url)
                decoded_content = download.content.decode('utf-8')
                cr = csv.reader(decoded_content.splitlines(),delimiter=',')
                for i in 10:
                    temp = cr[i]
            await self.bot.say(temp)
            await self.bot.say('DEBUG: CSV file opened')

#-------------- setup -------------
def check_folders():
    if not os.path.exists("data/clanmod"):
        print("Creating data/clanmod folder...")
        os.makedirs("data/clanmod")

    if not os.path.exists("data/clanmod/users"):
        print("Creating data/clanmod/users folder...")
        os.makedirs("data/clanmod/users")
        transfer_info()

def setup(bot):
    check_folders()
    n = ClanMod(bot)
    bot.add_cog(n)
    bot.add_listener(n.on_attachment, name='on_message')
<|endoftext|>