<commit_before>import BaseHTTPServer
import threading
from SimpleHTTPServer import SimpleHTTPRequestHandler

HandlerClass = SimpleHTTPRequestHandler
ServerClass  = BaseHTTPServer.HTTPServer
Protocol     = "HTTP/1.0"

port = 8888
server_address = ('127.0.0.1', port)

HandlerClass.protocol_version = Protocol
httpd = ServerClass(server_address, HandlerClass)

sa = httpd.socket.getsockname()

httpd_thread = threading.Thread(target=httpd.serve_forever)
httpd_thread.setDaemon(True)
print "Serving HTTP on", sa[0], "port", sa[1], "..."
httpd_thread.start()

from unittest import TestCase
import tempfile
import requests
import csv
import sys
from StringIO import StringIO
sort_by_url = lambda k: k['url']

class FunctionalTests(TestCase):
    base = 'http://localhost:%d/tests/html-files' % port

    def setUp(self):
        self.tmp = tempfile.TemporaryFile()

    def tearDown(self):
        self.tmp.close()

    def test_testsetup(self):
        assert requests.get(self.base).status_code == 200

    def test_metaspider(self):
        from eek import spider
        pages = [dict([(h, '') for h in spider.headers]) for x in range(4)]
        for p in pages:
            p['referer'] = self.base + '/index.html'
            p['status'] = '200'
        pages[0]['url'] = self.base + '/index.html'
        pages[0]['keywords'] = 'my_keywords'
        pages[0]['title'] = 'Index'
        pages[0]['description'] = 'my_description'
        pages[0]['meta robots'] = 'arst'
        pages[1]['url'] = self.base + '/a.html'
        pages[2]['url'] = self.base + '/b.html'
        pages[3]['url'] = self.base + '/c.html'
        pages = sorted(pages, key=sort_by_url)
        spider.metadata_spider(self.base + '/index.html', output=self.tmp)
        self.tmp.seek(0)
        result = sorted(list(csv.DictReader(self.tmp)), key=sort_by_url)
        self.assertEquals(pages, result)

    def test_grepspider(self):
        from eek import spider
        sys.stdout = StringIO()
        spider.grep_spider(self.base + '/index.html', 'Found me')
        self.assertEquals(sys.stdout.getvalue(), self.base + '/b.html:Found me\n')

    def test_graphvizspider(self):
        from eek import spider
        sys.stdout = StringIO()
        spider.graphviz_spider(self.base + '/index.html')
        graphiz_assertion = """digraph links {{
  "{base}/index.html" -> "{base}/a.html";
  "{base}/index.html" -> "{base}/b.html";
  "{base}/index.html" -> "{base}/c.html";
}}
""".format(base=self.base)
        self.assertEquals(sys.stdout.getvalue(), graphiz_assertion)
<commit_msg>test suites for eek.spider `utility functions` and eek.spider.beautify<commit_after>import BaseHTTPServer
import threading
from SimpleHTTPServer import SimpleHTTPRequestHandler

HandlerClass = SimpleHTTPRequestHandler
ServerClass = BaseHTTPServer.HTTPServer
Protocol = "HTTP/1.0"

port = 8888
server_address = ('127.0.0.1', port)

HandlerClass.protocol_version = Protocol
httpd = ServerClass(server_address, HandlerClass)

sa = httpd.socket.getsockname()

httpd_thread = threading.Thread(target=httpd.serve_forever)
httpd_thread.setDaemon(True)
print "Serving HTTP on", sa[0], "port", sa[1], "..."
httpd_thread.start()

from unittest import TestCase
import tempfile
import requests
import csv
import sys
from StringIO import StringIO
sort_by_url = lambda k: k['url']

class TestBeautify(TestCase):
    class MockResponse(object):
        def __init__(self, content_type, content):
            self.headers = {'content-type': content_type}
            self.content = content

    def test_return_BeautifulSoup_instance(self):
        from eek.spider import beautify, html_re
        from eek.BeautifulSoup import BeautifulSoup
        response = self.MockResponse(html_re.pattern,
                                     '<!doctype html>
Hello
')
        result = beautify(response)
        self.assertIsInstance(result, BeautifulSoup)

    def test_return_no_encoding(self):
        from eek.spider import beautify
        response = self.MockResponse('text/html',
                                     '<!doctype html>
Hello
')
        result = beautify(response)
        self.assertIsNone(result.fromEncoding)

    def test_return_not_none_encoding(self):
        from eek.spider import beautify
        response = self.MockResponse('text/html; charset=utf-8',
                                     '<!doctype html>
Hello
')
        result = beautify(response)
        self.assertIsNotNone(result.fromEncoding)

    def test_raise_NotHtmlException_on_messed_up_content_type(self):
        from eek.spider import beautify, NotHtmlException
        response = self.MockResponse('foo',
                                     '<!doctype html>
Hello
')
        with self.assertRaises(NotHtmlException):
            beautify(response)

class TestSpiderUtilities(TestCase):

    def test_encoding_from_content_type_none(self):
        from eek.spider import encoding_from_content_type
        self.assertIsNone(encoding_from_content_type('text/html'))

    def test_encoding_from_content_type_with_charset(self):
        from eek.spider import encoding_from_content_type
        self.assertEquals(
            encoding_from_content_type('text/html; charset=utf-8'),
            'utf-8')

    def test_lremove_prefix_exists(self):
        from eek.spider import lremove
        self.assertEquals(lremove('www.foo.com', 'www.'), 'foo.com')

    def test_lremove_prefix_not_exists(self):
        from eek.spider import lremove
        self.assertEquals(lremove('www.foo.com', 'eggs'), 'www.foo.com')

    def test_force_unicode_is_string_instance(self):
        from eek.spider import force_unicode
        uni_string = force_unicode('spam')
        self.assertIsInstance(uni_string, unicode)
        self.assertEquals(uni_string, u'spam')

    def test_force_unicode_not_string_instance(self):
        from eek.spider import force_unicode
        result = force_unicode(5)
        self.assertNotIsInstance(result, unicode)
        self.assertEquals(result, 5)
        result = force_unicode(None)
        self.assertNotIsInstance(result, unicode)
        self.assertEquals(result, None)

    def test_force_bytes_unicode_literal(self):
        from eek.spider import force_bytes
        self.assertEquals(force_bytes(u'\u2603'), '\xe2\x98\x83')

    def test_force_bytes_string(self):
        from eek.spider import force_bytes
        self.assertEquals(force_bytes('\xe2\x98\x83'), '\xe2\x98\x83')

class FunctionalTests(TestCase):
    base = 'http://localhost:%d/tests/html-files' % port

    def setUp(self):
        self.tmp = tempfile.TemporaryFile()

    def tearDown(self):
        self.tmp.close()

    def test_testsetup(self):
        assert requests.get(self.base).status_code == 200

    def test_metaspider(self):
        from eek import spider
        pages = [dict([(h, '') for h in spider.headers]) for x in range(4)]
        for p in pages:
            p['referer'] = self.base + '/index.html'
            p['status'] = '200'
        pages[0]['url'] = self.base + '/index.html'
        pages[0]['keywords'] = 'my_keywords'
        pages[0]['title'] = 'Index'
        pages[0]['description'] = 'my_description'
        pages[0]['meta robots'] = 'arst'
        pages[1]['url'] = self.base + '/a.html'
        pages[2]['url'] = self.base + '/b.html'
        pages[3]['url'] = self.base + '/c.html'
        pages = sorted(pages, key=sort_by_url)
        spider.metadata_spider(self.base + '/index.html', output=self.tmp)
        self.tmp.seek(0)
        result = sorted(list(csv.DictReader(self.tmp)), key=sort_by_url)
        self.assertEquals(pages, result)

    def test_grepspider(self):
        from eek import spider
        sys.stdout = StringIO()
        spider.grep_spider(self.base + '/index.html', 'Found me')
        self.assertEquals(sys.stdout.getvalue(), self.base + '/b.html:Found me\n')

    def test_graphvizspider(self):
        from eek import spider
        sys.stdout = StringIO()
        spider.graphviz_spider(self.base + '/index.html')
        graphiz_assertion = """digraph links {{
  "{base}/index.html" -> "{base}/a.html";
  "{base}/index.html" -> "{base}/b.html";
  "{base}/index.html" -> "{base}/c.html";
}}
""".format(base=self.base)
        self.assertEquals(sys.stdout.getvalue(), graphiz_assertion)
<|endoftext|>


<commit_before>from django.contrib.auth.decorators import login_required
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import render
from django.utils.datetime_safe import datetime
from django.utils.decorators import method_decorator
from django.views.generic import ListView
from django.http import HttpResponseRedirect, JsonResponse
from buzzit_messaging.views import __send_system__message__
from buzzit_models.models import *
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse, reverse_lazy
import django.contrib.messages as messages
from django.views.generic import ListView
from buzzit_models.models import *
from django.contrib.auth.decorators import login_required
import json

@login_required()
def report_user(request,user_id):
    """
    current user report other user, and gives the reason which would not be empty
    :param request:
    :param user_id:
    :return:
    """

    try:
        reported_user = User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Benutzer existiert nicht.")
        return HttpResponseRedirect(reverse_lazy("home"))
    if request.method == "POST":
        report_text = request.POST.get["text"]
        if not report_text:
            messages.error(request,"Text zum Benutzermelden ist zu geben")
            return HttpResponseRedirect(reverse_lazy("home"))
        report_message = Report(creator=request.user,created=datetime.now(),text=report_text)
        report_message.save()
        messages.INFO("Sie haben den <User:%s> Benutzer gemeldet" %reported_user)

    # send notifications to system users
    system_user = User.objects.get(username="SYSTEM")
    __send_system__message__(system_user.pk, "<Report:%s> Neue Meldung " % report_message)

    return HttpResponseRedirect(reverse_lazy('home'))

class UserReportDetailsView(SuccessMessageMixin,ListView):
    """
    display the report text and reported user
    """
    model = Report
    template_name = "logged_in/user_report_deatails"
    context_object_name = "all_reports"

    def get_queryset(self):
        pass

    def get_context_data(self, **kwargs):
        pass

class AdminFrontpageView():
    pass

class MessageReportDetailsView(ListView):
    pass

class AdminOverviewView(ListView):
    pass

@login_required
def delete_reported_post(request, message_id):
    pass

@login_required
def promote_user_to_admin(request, user_id):
    pass

@login_required
def demote_admin_to_user(request, user_id):
    pass

@login_required
def report_message(request, message_id):
    pass

@login_required
def ban_user(request, user_id):
    pass
<commit_msg>change<commit_after>from django.contrib.auth.decorators import login_required
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import render
from django.utils.datetime_safe import datetime
from django.utils.decorators import method_decorator
from django.views.generic import ListView
from django.http import HttpResponseRedirect, JsonResponse
from buzzit_messaging.views import __send_system__message__
from buzzit_models.models import *
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse, reverse_lazy
import django.contrib.messages as messages
from django.views.generic import ListView
from buzzit_models.models import *
from django.contrib.auth.decorators import login_required
import json

@login_required()
def report_user(request,user_id):
    """
    current user report other user, and gives the reason which would not be empty
    :param request:
    :param user_id:
    :return:
    """

    try:
        reported_user = User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Benutzer existiert nicht.")
        return HttpResponseRedirect(reverse_lazy("home"))
    if request.method == "POST":
        report_text = request.POST.get["text"]
        if not report_text:
            messages.error(request,"Text zum Benutzermelden ist zu geben")
            return HttpResponseRedirect(reverse_lazy("home"))
        report_message = UserReport(creator=request.user,created=datetime.now(),text=report_text,reported_user=reported_user)
        report_message.save()
        messages.info("Sie haben den <User:%s> Benutzer gemeldet" %reported_user)

    # send notifications to system users
    system_user = User.objects.get(username="SYSTEM")
    __send_system__message__(system_user.pk, "<Report:%s> Neue Meldung " % report_message)

    return HttpResponseRedirect(reverse_lazy('home'))

class UserReportDetailsView(SuccessMessageMixin,ListView):
    """
    display the report text and reported user
    """
    model = Report
    template_name = "logged_in/user_report_deatails"

    def get_queryset(self):
        pass

    def get_context_data(self, **kwargs):
        pass

class AdminFrontpageView():
    pass

class MessageReportDetailsView(ListView):
    pass

class AdminOverviewView(ListView):
    pass

@login_required
def delete_reported_post(request, message_id):
    pass

@login_required
def promote_user_to_admin(request, user_id):
    pass

@login_required
def demote_admin_to_user(request, user_id):
    pass

@login_required
def report_message(request, message_id):
    pass

@login_required
def ban_user(request, user_id):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import unittest
from collections import Counter

from pythainlp.change import texttoeng, texttothai
from pythainlp.collation import collation
from pythainlp.corpus import (
    alphabet,
    country,
    newthaiword,
    provinces,
    stopwords,
    thaiword,
    tone,
    wordnet,
)
from pythainlp.date import now, reign_year_to_ad, now_reign_year
from pythainlp.keywords import find_keyword
from pythainlp.MetaSound import MetaSound
from pythainlp.ner import ThaiNameRecognizer
from pythainlp.number import numtowords
from pythainlp.rank import rank
from pythainlp.romanization import romanize
from pythainlp.soundex import LK82, Udom83
from pythainlp.spell import spell
from pythainlp.summarize import summarize_text
from pythainlp.tag import pos_tag, pos_tag_sents
from pythainlp.tokenize import etcc, isthai, syllable_tokenize, tcc, word_tokenize
from pythainlp.util import listtext_num2num, normalize
from pythainlp.Text import Text

class TestUM(unittest.TestCase):
    """
    ทดสอบการทำงาน
    """

    def test_segment(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_syllable_tokenize(self):
        self.assertEqual(
            syllable_tokenize("สวัสดีชาวโลก"), ["สวัส", "ดี", "ชาว", "โลก"]
        )

    def test_segment_icu(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"),
            ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
        )

    def test_segment_mm(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="mm"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_segment_newmm(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="newmm"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )
        self.assertEqual(
            word_tokenize("สวัสดีครับ สบายดีไหมครับ", engine="newmm"),
            ["สวัสดี", "ครับ", " ", "สบายดี", "ไหม", "ครับ"],
        )
        self.assertEqual(
            word_tokenize("จุ๋มง่วงนอนยัง", engine="newmm"), ["จุ๋ม", "ง่วงนอน", "ยัง"]
        )
        self.assertEqual(word_tokenize("จุ๋มง่วง", engine="newmm"), ["จุ๋ม", "ง่วง"])
        self.assertEqual(
            word_tokenize("จุ๋ม   ง่วง", engine="newmm", whitespaces=False),
            ["จุ๋ม", "ง่วง"],
        )

    def test_segment_longest_matching(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="longest-matching"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_segment_wordcut(self):
        if (
            sys.version_info >= (3, 4)
            and sys.platform != "win32"
            and sys.platform != "win64"
        ):
            self.assertEqual(
                word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="wordcutpy"),
                ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
            )

    def test_rank(self):
        self.assertEqual(rank(["แมว", "คน", "แมว"]), Counter({"แมว": 2, "คน": 1}))

    def test_change(self):
        self.assertEqual(texttothai("l;ylfu8iy["), "สวัสดีครับ")
        self.assertEqual(texttoeng("สวัสดีครับ"), "l;ylfu8iy[")

    def test_romanization(self):
        self.assertEqual(romanize("แมว"), "maeo")
        self.assertEqual(romanize("แมว", "pyicu"), "mæw")

    def test_romanization_royin(self):
        engine = "royin"
        self.assertEqual(romanize("แมว", engine=engine), "maeo")
        self.assertEqual(romanize("เดือน", engine=engine), "duean")
        self.assertEqual(romanize("ดู", engine=engine), "du")
        self.assertEqual(romanize("ดำ", engine=engine), "dam")
        self.assertEqual(romanize("บัว", engine=engine), "bua")

    def test_number(self):
        self.assertEqual(
            numtowords(5611116.50),
            "ห้าล้านหกแสนหนึ่งหมื่นหนึ่งพันหนึ่งร้อยสิบหกบาทห้าสิบสตางค์",
        )

    def test_tcc(self):
        self.assertEqual(tcc.tcc("ประเทศไทย"), "ป/ระ/เท/ศ/ไท/ย")

    def test_isthai(self):
        self.assertEqual(isthai("ประเทศไทย"), {"thai": 100.0})

    # def test_WhitespaceTokenizer(self):
    #     self.assertEqual(WhitespaceTokenizer("1 2 3"),['1', '2', '3'])

    def test_etcc(self):
        self.assertEqual(etcc.etcc("คืนความสุข"), "/คืน/ความสุข")

    def test_lk82(self):
        self.assertEqual(LK82("รถ"), "ร3000")
        self.assertEqual(Udom83("รถ"), "ร800000")

    def test_ms(self):
        self.assertEqual(MetaSound("คน"), "15")

    def test_wordnet(self):
        self.assertEqual(
            wordnet.synset("spy.n.01").lemma_names("tha"), ["สปาย", "สายลับ"]
        )
        self.assertIsNotNone(wordnet.langs())

    def test_stopword(self):
        self.assertIsNotNone(stopwords.words("thai"))

    def test_spell(self):
        self.assertIsNotNone(spell("เน้ร"))

    def test_date(self):
        self.assertIsNotNone(now())
        self.assertEqual(reign_year_to_ad(2, 10), 2017)
        self.assertIsNotNone(reign_year_to_ad(2, 9))
        self.assertIsNotNone(reign_year_to_ad(2, 8))
        self.assertIsNotNone(reign_year_to_ad(2, 7))
        self.assertIsNotNone(now_reign_year())

    def test_summarize(self):
        text = "อาหาร หมายถึง ของแข็งหรือของเหลว "
        text += "ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว "
        text += "จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย "
        text += "ทำให้ร่างกายเจริญเติบโต "
        text += "ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย "
        text += "ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ "
        text += "อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย"
        self.assertEqual(
            summarize_text(text=text, n=1, engine="frequency"),
            ["อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย"],
        )

    def test_corpus(self):
        self.assertIsNotNone(alphabet.get_data())
        self.assertIsNotNone(country.get_data())
        self.assertIsNotNone(tone.get_data())
        self.assertIsNotNone(provinces.get_data())
        self.assertTrue(len(newthaiword.get_data()) > len(thaiword.get_data()))

    def test_collation(self):
        self.assertEqual(collation(["ไก่", "กก"]), ["กก", "ไก่"])
        self.assertEqual(
            collation(["ไก่", "เป็ด", "หมู", "วัว"]), ["ไก่", "เป็ด", "วัว", "หมู"]
        )

    def test_normalize(self):
        self.assertEqual(normalize("เเปลก"), "แปลก")

    def test_listtext_num2num(self):
        if sys.version_info >= (3, 4):
            self.assertEqual(
                listtext_num2num(
                    ["หก", "ล้าน", "หกแสน", "หกหมื่น", "หกพัน", "หกร้อย", "หกสิบ", "หก"]
                ),
                6666666,
            )

    def test_keywords(self):
        word_list = word_tokenize(
            "แมวกินปลาอร่อยรู้ไหมว่าแมวเป็นแมวรู้ไหมนะแมว", engine="newmm"
        )
        self.assertEqual(find_keyword(word_list), {"แมว": 4})

    def test_tag(self):
        self.assertEqual(
            pos_tag(word_tokenize("คุณกำลังประชุม"), engine="old"),
            [("คุณ", "PPRS"), ("กำลัง", "XVBM"), ("ประชุม", "VACT")],
        )
        self.assertEqual(
            pos_tag_sents([["ผม", "กิน", "ข้าว"], ["แมว", "วิ่ง"]]),
            [
                [("ผม", "PPRS"), ("กิน", "VACT"), ("ข้าว", "NCMN")],
                [("แมว", "NCMN"), ("วิ่ง", "VACT")],
            ],
        )

        if sys.version_info >= (3, 4):
            self.assertEqual(
                str(type(pos_tag(word_tokenize("ผมรักคุณ"), engine="artagger"))),
                "<class 'list'>",
            )

    def test_ner(self):
        ner = ThaiNameRecognizer()
        self.assertEqual(
            ner.get_ner("แมวทำอะไรตอนห้าโมงเช้า"),
            [
                ("แมว", "NCMN", "O"),
                ("ทำ", "VACT", "O"),
                ("อะไร", "PNTR", "O"),
                ("ตอน", "NCMN", "O"),
                ("ห้า", "VSTA", "B-TIME"),
                ("โมง", "NCMN", "I-TIME"),
                ("เช้า", "ADVN", "I-TIME"),
            ],
        )
        self.assertEqual(
            ner.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False),
            [
                ("แมว", "O"),
                ("ทำ", "O"),
                ("อะไร", "O"),
                ("ตอน", "O"),
                ("ห้า", "B-TIME"),
                ("โมง", "I-TIME"),
                ("เช้า", "I-TIME"),
            ],
        )
    def test_Text(self):
        self.assertIsNotNone(Text("ทดสอบภาษาไทย"))

if __name__ == "__main__":
    unittest.main()
<commit_msg>Add more MetaSound test<commit_after># -*- coding: utf-8 -*-
import sys
import unittest
from collections import Counter

from pythainlp.change import texttoeng, texttothai
from pythainlp.collation import collation
from pythainlp.corpus import (
    alphabet,
    country,
    newthaiword,
    provinces,
    stopwords,
    thaiword,
    tone,
    wordnet,
)
from pythainlp.date import now, reign_year_to_ad, now_reign_year
from pythainlp.keywords import find_keyword
from pythainlp.MetaSound import MetaSound
from pythainlp.ner import ThaiNameRecognizer
from pythainlp.number import numtowords
from pythainlp.rank import rank
from pythainlp.romanization import romanize
from pythainlp.soundex import LK82, Udom83
from pythainlp.spell import spell
from pythainlp.summarize import summarize_text
from pythainlp.tag import pos_tag, pos_tag_sents
from pythainlp.tokenize import etcc, isthai, syllable_tokenize, tcc, word_tokenize
from pythainlp.util import listtext_num2num, normalize
from pythainlp.Text import Text

class TestUM(unittest.TestCase):
    """
    ทดสอบการทำงาน
    """

    def test_segment(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_syllable_tokenize(self):
        self.assertEqual(
            syllable_tokenize("สวัสดีชาวโลก"), ["สวัส", "ดี", "ชาว", "โลก"]
        )

    def test_segment_icu(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"),
            ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
        )

    def test_segment_mm(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="mm"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_segment_newmm(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="newmm"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )
        self.assertEqual(
            word_tokenize("สวัสดีครับ สบายดีไหมครับ", engine="newmm"),
            ["สวัสดี", "ครับ", " ", "สบายดี", "ไหม", "ครับ"],
        )
        self.assertEqual(
            word_tokenize("จุ๋มง่วงนอนยัง", engine="newmm"), ["จุ๋ม", "ง่วงนอน", "ยัง"]
        )
        self.assertEqual(word_tokenize("จุ๋มง่วง", engine="newmm"), ["จุ๋ม", "ง่วง"])
        self.assertEqual(
            word_tokenize("จุ๋ม   ง่วง", engine="newmm", whitespaces=False),
            ["จุ๋ม", "ง่วง"],
        )

    def test_segment_longest_matching(self):
        self.assertEqual(
            word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="longest-matching"),
            ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
        )

    def test_segment_wordcut(self):
        if (
            sys.version_info >= (3, 4)
            and sys.platform != "win32"
            and sys.platform != "win64"
        ):
            self.assertEqual(
                word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="wordcutpy"),
                ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
            )

    def test_rank(self):
        self.assertEqual(rank(["แมว", "คน", "แมว"]), Counter({"แมว": 2, "คน": 1}))

    def test_change(self):
        self.assertEqual(texttothai("l;ylfu8iy["), "สวัสดีครับ")
        self.assertEqual(texttoeng("สวัสดีครับ"), "l;ylfu8iy[")

    def test_romanization(self):
        self.assertEqual(romanize("แมว"), "maeo")
        self.assertEqual(romanize("แมว", "pyicu"), "mæw")

    def test_romanization_royin(self):
        engine = "royin"
        self.assertEqual(romanize("แมว", engine=engine), "maeo")
        self.assertEqual(romanize("เดือน", engine=engine), "duean")
        self.assertEqual(romanize("ดู", engine=engine), "du")
        self.assertEqual(romanize("ดำ", engine=engine), "dam")
        self.assertEqual(romanize("บัว", engine=engine), "bua")

    def test_number(self):
        self.assertEqual(
            numtowords(5611116.50),
            "ห้าล้านหกแสนหนึ่งหมื่นหนึ่งพันหนึ่งร้อยสิบหกบาทห้าสิบสตางค์",
        )

    def test_tcc(self):
        self.assertEqual(tcc.tcc("ประเทศไทย"), "ป/ระ/เท/ศ/ไท/ย")

    def test_isthai(self):
        self.assertEqual(isthai("ประเทศไทย"), {"thai": 100.0})

    def test_etcc(self):
        self.assertEqual(etcc.etcc("คืนความสุข"), "/คืน/ความสุข")

    def test_lk82(self):
        self.assertEqual(LK82("รถ"), "ร3000")
        self.assertEqual(Udom83("รถ"), "ร800000")

    def test_ms(self):
        self.assertEqual(MetaSound("คน"), "15")
        self.assertEqual(MetaSound("คนA"), "150")
        self.assertEqual(MetaSound("ดา"), "20")
        self.assertEqual(MetaSound("ปา"), "30")
        self.assertEqual(MetaSound("งา"), "40")
        self.assertEqual(MetaSound("ลา"), "50")
        self.assertEqual(MetaSound("มา"), "60")
        self.assertEqual(MetaSound("วา"), "80")
        self.assertEqual(MetaSound("ลัก"), MetaSound("รัก"))

    def test_wordnet(self):
        self.assertEqual(
            wordnet.synset("spy.n.01").lemma_names("tha"), ["สปาย", "สายลับ"]
        )
        self.assertIsNotNone(wordnet.langs())

    def test_stopword(self):
        self.assertIsNotNone(stopwords.words("thai"))

    def test_spell(self):
        self.assertIsNotNone(spell("เน้ร"))

    def test_date(self):
        self.assertIsNotNone(now())
        self.assertEqual(reign_year_to_ad(2, 10), 2017)
        self.assertIsNotNone(reign_year_to_ad(2, 9))
        self.assertIsNotNone(reign_year_to_ad(2, 8))
        self.assertIsNotNone(reign_year_to_ad(2, 7))
        self.assertIsNotNone(now_reign_year())

    def test_summarize(self):
        text = "อาหาร หมายถึง ของแข็งหรือของเหลว "
        text += "ที่กินหรือดื่มเข้าสู่ร่างกายแล้ว "
        text += "จะทำให้เกิดพลังงานและความร้อนแก่ร่างกาย "
        text += "ทำให้ร่างกายเจริญเติบโต "
        text += "ซ่อมแซมส่วนที่สึกหรอ ควบคุมการเปลี่ยนแปลงต่างๆ ในร่างกาย "
        text += "ช่วยทำให้อวัยวะต่างๆ ทำงานได้อย่างปกติ "
        text += "อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย"
        self.assertEqual(
            summarize_text(text=text, n=1, engine="frequency"),
            ["อาหารจะต้องไม่มีพิษและไม่เกิดโทษต่อร่างกาย"],
        )

    def test_corpus(self):
        self.assertIsNotNone(alphabet.get_data())
        self.assertIsNotNone(country.get_data())
        self.assertIsNotNone(tone.get_data())
        self.assertIsNotNone(provinces.get_data())
        self.assertTrue(len(newthaiword.get_data()) > len(thaiword.get_data()))

    def test_collation(self):
        self.assertEqual(collation(["ไก่", "กก"]), ["กก", "ไก่"])
        self.assertEqual(
            collation(["ไก่", "เป็ด", "หมู", "วัว"]), ["ไก่", "เป็ด", "วัว", "หมู"]
        )

    def test_normalize(self):
        self.assertEqual(normalize("เเปลก"), "แปลก")

    def test_listtext_num2num(self):
        if sys.version_info >= (3, 4):
            self.assertEqual(
                listtext_num2num(
                    ["หก", "ล้าน", "หกแสน", "หกหมื่น", "หกพัน", "หกร้อย", "หกสิบ", "หก"]
                ),
                6666666,
            )

    def test_keywords(self):
        word_list = word_tokenize(
            "แมวกินปลาอร่อยรู้ไหมว่าแมวเป็นแมวรู้ไหมนะแมว", engine="newmm"
        )
        self.assertEqual(find_keyword(word_list), {"แมว": 4})

    def test_tag(self):
        self.assertEqual(
            pos_tag(word_tokenize("คุณกำลังประชุม"), engine="old"),
            [("คุณ", "PPRS"), ("กำลัง", "XVBM"), ("ประชุม", "VACT")],
        )
        self.assertEqual(
            pos_tag_sents([["ผม", "กิน", "ข้าว"], ["แมว", "วิ่ง"]]),
            [
                [("ผม", "PPRS"), ("กิน", "VACT"), ("ข้าว", "NCMN")],
                [("แมว", "NCMN"), ("วิ่ง", "VACT")],
            ],
        )

        if sys.version_info >= (3, 4):
            self.assertEqual(
                str(type(pos_tag(word_tokenize("ผมรักคุณ"), engine="artagger"))),
                "<class 'list'>",
            )

    def test_ner(self):
        ner = ThaiNameRecognizer()
        self.assertEqual(
            ner.get_ner("แมวทำอะไรตอนห้าโมงเช้า"),
            [
                ("แมว", "NCMN", "O"),
                ("ทำ", "VACT", "O"),
                ("อะไร", "PNTR", "O"),
                ("ตอน", "NCMN", "O"),
                ("ห้า", "VSTA", "B-TIME"),
                ("โมง", "NCMN", "I-TIME"),
                ("เช้า", "ADVN", "I-TIME"),
            ],
        )
        self.assertEqual(
            ner.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False),
            [
                ("แมว", "O"),
                ("ทำ", "O"),
                ("อะไร", "O"),
                ("ตอน", "O"),
                ("ห้า", "B-TIME"),
                ("โมง", "I-TIME"),
                ("เช้า", "I-TIME"),
            ],
        )
    def test_Text(self):
        self.assertIsNotNone(Text("ทดสอบภาษาไทย"))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2006,2008 Herbert Valerio Riedel <hvr@gnu.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

import os, re
#from traceback import print_stack

class GitError(Exception):
    pass

class GitErrorSha(GitError):
    pass

class Storage:
    def __init__(self,repo):
        self.repo = repo
        self.commit_encoding = None

    def _git_call_f(self,cmd):
        #print "GIT: "+cmd
        (input, output, error) = os.popen3('GIT_DIR="%s" %s' % (self.repo,cmd))
        return output

    def _git_call(self,cmd):
        return self._git_call_f(cmd).read()

    def get_commit_encoding(self):
        if self.commit_encoding is None:
            self.commit_encoding = self._git_call("git-repo-config --get i18n.commitEncoding").strip()
            if ''==self.commit_encoding:
                self.commit_encoding = 'utf-8'
        return self.commit_encoding

    def head(self):
        "get current HEAD commit id"
        return self.verifyrev("HEAD")

    def verifyrev(self,rev):
        "verify/lookup given revision object and return a sha id or None if lookup failed"
        rc=self._git_call("git-rev-parse --verify '%s'" % rev).strip()
        if len(rc)==0:
            return None
        return rc

    def shortrev(self,rev):
        "try to shorten sha id"
        return self._git_call("git-rev-parse --short '%s'" % rev).strip()

    def get_branches(self):
        "returns list of branches, with active (i.e. HEAD) one being the first item"
        result=[]
        for e in self._git_call_f("git-branch").readlines():
            bname=e[1:].strip()
            if e[0]=='*':
                result.insert(0,bname)
            else:
                result.append(bname)
        return result

    def tree_ls(self,sha,path=""):
        if len(path)>0 and path[0]=='/':
            path=path[1:]
        return [e[:-1].split(None, 3) for e in self._git_call_f("git-ls-tree %s '%s'" % (sha,path)).readlines()]

    def read_commit(self, sha):
        raw = self._git_call("git-cat-file commit "+sha)
        raw = unicode(raw, self.get_commit_encoding(), 'replace')
        lines = raw.splitlines()

        if not lines:
            raise GitErrorSha

        line = lines.pop(0)
        d = {}
        while line != "":
            (key,value)=line.split(None, 1)
            if not d.has_key(key):
                d[key] = []
            d[key].append(value.strip())
            line = lines.pop(0)

        return ("\n".join(lines),d)

    def get_file(self, sha):
        return self._git_call_f("git-cat-file blob "+sha)

    def get_obj_size(self, sha):
        return int(self._git_call("git-cat-file -s "+sha).strip())

    def parents(self, sha):
        tmp=self._git_call("git-rev-list --max-count=1 --parents "+sha)
        tmp=tmp.strip()
        tmp=tmp.split()
        return tmp[1:]

    def children(self, sha):
        for revs in self._git_call_f("git-rev-list --parents HEAD").readlines():
            revs = revs.strip()
            revs = revs.split()
            if sha in revs[1:]:
                yield revs[0]

    def history(self, sha, path, limit=None, skip=0):
        #print "history", sha, path, limit, skip
        if limit is None:
            limit = -1
        for rev in self._git_call_f("git-rev-list -n%d %s -- '%s'" % (limit,sha,path)).readlines():
            if(skip > 0):
                skip = skip - 1
                continue
            yield rev.strip()

    def history_all(self, start, stop):
        for rev in self._git_call_f("git-rev-list --reverse --max-age=%d --min-age=%d --all" \
                                        % (start,stop)).readlines():
            yield rev.strip()

    def rev_is_anchestor(self, rev1, rev2):
        rev1 = rev1.strip()
        rev2 = rev2.strip()
        for rev in self._git_call_f("git-rev-list %s ^%s^" % (rev2,rev1)).readlines():
            if rev1 == rev.strip():
                return True
        return False

    def last_change(self, sha, path):
        for rev in self._git_call_f("git-rev-list --max-count=1 %s -- '%s'" % (sha,path)).readlines():
            return rev.strip()
        return None

    def diff_tree(self, tree1, tree2, path=""):
        if tree1 is None:
            tree1 = "--root"
        cmd = "git-diff-tree -r %s %s -- '%s'" % (tree1, tree2, path)
        for chg in self._git_call_f(cmd).readlines():
            if chg.startswith(tree2):
                continue
            (mode1,mode2,obj1,obj2,action,path) = chg[:-1].split(None, 5)
            mode1 = mode1[1:]
            yield (mode1,mode2,obj1,obj2,action,path)

if __name__ == '__main__':
    import sys

    g = Storage(sys.argv[1])

    print "[%s]" % g.head()
    print g.tree_ls(g.head())
    print "--------------"
    print g.read_commit(g.head())
    print "--------------"
    print g.parents(g.head())

    print "--------------"
    print g.get_commit_encoding()
    print "--------------"
    print g.get_branches()
<commit_msg>added code to aid profiling expensive git operations<commit_after># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2006,2008 Herbert Valerio Riedel <hvr@gnu.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

import os, re, sys, time
#from traceback import print_stack

_profile_git_calls = False

class GitError(Exception):
    pass

class GitErrorSha(GitError):
    pass

class Storage:
    def __init__(self,repo):
        self.repo = repo
        self.commit_encoding = None

    def _git_call_f(self,cmd):
        #print "GIT: "+cmd
        if _profile_git_calls:
            t = time.time()
            pass

        (input, output, error) = os.popen3('GIT_DIR="%s" %s' % (self.repo,cmd))

        if _profile_git_calls:
            t = time.time() - t
            print >>sys.stderr, "GIT: took %6.2fs for '%s'" % (t, cmd)
            pass

        return output

    def _git_call(self,cmd):
        return self._git_call_f(cmd).read()

    def get_commit_encoding(self):
        if self.commit_encoding is None:
            self.commit_encoding = self._git_call("git-repo-config --get i18n.commitEncoding").strip()
            if ''==self.commit_encoding:
                self.commit_encoding = 'utf-8'
        return self.commit_encoding

    def head(self):
        "get current HEAD commit id"
        return self.verifyrev("HEAD")

    def verifyrev(self,rev):
        "verify/lookup given revision object and return a sha id or None if lookup failed"
        rc=self._git_call("git-rev-parse --verify '%s'" % rev).strip()
        if len(rc)==0:
            return None
        return rc

    def shortrev(self,rev):
        "try to shorten sha id"
        return self._git_call("git-rev-parse --short '%s'" % rev).strip()

    def get_branches(self):
        "returns list of branches, with active (i.e. HEAD) one being the first item"
        result=[]
        for e in self._git_call_f("git-branch").readlines():
            bname=e[1:].strip()
            if e[0]=='*':
                result.insert(0,bname)
            else:
                result.append(bname)
        return result

    def tree_ls(self,sha,path=""):
        if len(path)>0 and path[0]=='/':
            path=path[1:]
        return [e[:-1].split(None, 3) for e in self._git_call_f("git-ls-tree %s '%s'" % (sha,path)).readlines()]

    def read_commit(self, sha):
        raw = self._git_call("git-cat-file commit "+sha)
        raw = unicode(raw, self.get_commit_encoding(), 'replace')
        lines = raw.splitlines()

        if not lines:
            raise GitErrorSha

        line = lines.pop(0)
        d = {}
        while line != "":
            (key,value)=line.split(None, 1)
            if not d.has_key(key):
                d[key] = []
            d[key].append(value.strip())
            line = lines.pop(0)

        return ("\n".join(lines),d)

    def get_file(self, sha):
        return self._git_call_f("git-cat-file blob "+sha)

    def get_obj_size(self, sha):
        return int(self._git_call("git-cat-file -s "+sha).strip())

    def parents(self, sha):
        tmp=self._git_call("git-rev-list --max-count=1 --parents "+sha)
        tmp=tmp.strip()
        tmp=tmp.split()
        return tmp[1:]

    def children(self, sha):
        for revs in self._git_call_f("git-rev-list --parents HEAD").readlines():
            revs = revs.strip()
            revs = revs.split()
            if sha in revs[1:]:
                yield revs[0]

    def history(self, sha, path, limit=None, skip=0):
        #print "history", sha, path, limit, skip
        if limit is None:
            limit = -1
        for rev in self._git_call_f("git-rev-list -n%d %s -- '%s'" % (limit,sha,path)).readlines():
            if(skip > 0):
                skip = skip - 1
                continue
            yield rev.strip()

    def history_all(self, start, stop):
        for rev in self._git_call_f("git-rev-list --reverse --max-age=%d --min-age=%d --all" \
                                        % (start,stop)).readlines():
            yield rev.strip()

    def rev_is_anchestor(self, rev1, rev2):
        rev1 = rev1.strip()
        rev2 = rev2.strip()
        for rev in self._git_call_f("git-rev-list %s ^%s^" % (rev2,rev1)).readlines():
            if rev1 == rev.strip():
                return True
        return False

    def last_change(self, sha, path):
        for rev in self._git_call_f("git-rev-list --max-count=1 %s -- '%s'" % (sha,path)).readlines():
            return rev.strip()
        return None

    def diff_tree(self, tree1, tree2, path=""):
        if tree1 is None:
            tree1 = "--root"
        cmd = "git-diff-tree -r %s %s -- '%s'" % (tree1, tree2, path)
        for chg in self._git_call_f(cmd).readlines():
            if chg.startswith(tree2):
                continue
            (mode1,mode2,obj1,obj2,action,path) = chg[:-1].split(None, 5)
            mode1 = mode1[1:]
            yield (mode1,mode2,obj1,obj2,action,path)

if __name__ == '__main__':
    import sys

    g = Storage(sys.argv[1])

    print "[%s]" % g.head()
    print g.tree_ls(g.head())
    print "--------------"
    print g.read_commit(g.head())
    print "--------------"
    print g.parents(g.head())

    print "--------------"
    print g.get_commit_encoding()
    print "--------------"
    print g.get_branches()
<|endoftext|>


<commit_before>import sublime
import sublime_plugin
import gscommon as gs
import re
import os
import httplib
import hashlib
import threading
import Queue
import traceback
import subprocess
import time
import signal

DOMAIN = "GsShell"
GO_RUN_PAT = re.compile(r'^go\s+(run|play)$', re.IGNORECASE)
GO_SHARE_PAT = re.compile(r'^go\s+share$', re.IGNORECASE)
GO_PLAY_PAT = re.compile(r'(\b)go\s+play(\b)', re.IGNORECASE)

class Prompt(object):
	def __init__(self, view, change_history=True):
		self.view = view
		self.panel = None
		self.subcommands = [
			'go run', 'go build', 'go clean', 'go fix',
			'go install', 'go test', 'go fmt', 'go vet', 'go tool',
			'go share', 'go play',
		]
		self.settings = sublime.load_settings('GoSublime-GsShell.sublime-settings')
		self.change_history = change_history

	def on_done(self, s, fmt_save=True):
		fn = self.view.file_name()
		win = self.view.window()
		if fn and win:
			basedir = os.path.dirname(fn)
			if fmt_save:
				for v in win.views():
					vfn = v.file_name()
					if vfn and v.is_dirty() and os.path.dirname(vfn) == basedir and vfn.endswith('.go'):
						v.run_command('gs_fmt_save')
		elif fmt_save:
			self.view.run_command('gs_fmt')

		# above we do some saves - thus creating a race so push this back to the end of the queue
		def cb(s):
			file_name = self.view.file_name() or ''
			s = GO_PLAY_PAT.sub(r'\1go run\2', s)
			s = s.strip()
			if s and s.lower() != "go" and self.change_history:
				hist = self.settings.get('cmd_hist')
				if not gs.is_a(hist, {}):
					hist = {}
				basedir = gs.basedir_or_cwd(file_name)
				hist[basedir] = [s] # todo: store a list of historical commands
				hst = {}
				for k in hist:
					# :|
					hst[gs.ustr(k)] = gs.ustr(hist[k])
				self.settings.set('cmd_hist', hst)
				sublime.save_settings('GoSublime-GsShell.sublime-settings')

			if GO_SHARE_PAT.match(s):
				s = ''
				host = "play.golang.org"
				warning = 'Are you sure you want to share this file. It will be public on %s' % host
				if not sublime.ok_cancel_dialog(warning):
					return

				try:
					c = httplib.HTTPConnection(host)
					src = gs.astr(self.view.substr(sublime.Region(0, self.view.size())))
					c.request('POST', '/share', src, {'User-Agent': 'GoSublime'})
					s = 'http://%s/p/%s' % (host, c.getresponse().read())
				except Exception as ex:
					s = 'Error: %s' % ex

				self.show_output(s, focus=True)
				return

			if GO_RUN_PAT.match(s):
				if not file_name:
					# todo: clean this up after the command runs
					err = ''
					tdir, _ = gs.temp_dir('play')
					file_name = hashlib.sha1(gs.view_fn(self.view) or 'a').hexdigest()
					file_name = os.path.join(tdir, ('%s.go' % file_name))
					try:
						with open(file_name, 'w') as f:
							src = gs.astr(self.view.substr(sublime.Region(0, self.view.size())))
							f.write(src)
					except Exception as ex:
						err = str(ex)

					if err:
						self.show_output('Error: %s' % err)
						return

				s = ['go', 'run', file_name]

			self.view.window().run_command("exec", { 'kill': True })
			if gs.is_a(s, []):
				use_shell = False
			else:
				use_shell = True
				s = [s]
			gs.println('running %s' % ' '.join(s))
			self.view.window().run_command("exec", {
				'shell': use_shell,
				'env': gs.env(),
				'cmd': s,
				'file_regex': '^(.+\.go):([0-9]+):(?:([0-9]+):)?\s*(.*)',
			})

		sublime.set_timeout(lambda: cb(s), 0)

	def show_output(self, s, focus=False):
		panel_name = DOMAIN+'-share'
		win = self.view.window()
		panel = win.get_output_panel(panel_name)
		edit = panel.begin_edit()
		try:
			panel.set_read_only(False)
			panel.sel().clear()
			panel.replace(edit, sublime.Region(0, panel.size()), s)
			panel.sel().add(sublime.Region(0, panel.size()))
			panel.set_read_only(True)
		finally:
			panel.end_edit(edit)
		print('%s output: %s' % (DOMAIN, s))
		self.view.window().run_command("show_panel", {"panel": "output.%s" % panel_name})
		if focus:
			sublime.set_timeout(lambda: win.focus_view(panel), 0)

	def on_change(self, s):
		if self.panel:
			size = self.view.size()
			if s.endswith('\t'):
				basedir = gs.basedir_or_cwd(self.view.file_name())
				lc = 'go '
				hist = self.settings.get('cmd_hist')
				if gs.is_a(hist, {}):
					hist = hist.get(basedir)
					if hist and gs.is_a(hist, []):
						lc = hist[-1]
				s = s.strip()
				if s and s not in ('', 'go'):
					l = []
					for i in self.subcommands:
						if i.startswith(s):
							l.append(i)
					if len(l) == 1:
						s = '%s ' % l[0]
				elif lc:
					s = '%s ' % lc
				edit = self.panel.begin_edit()
				try:
					self.panel.replace(edit, sublime.Region(0, self.panel.size()), s)
				finally:
					self.panel.end_edit(edit)

class GsShellCommand(sublime_plugin.WindowCommand):
	def is_enabled(self):
		view = gs.active_valid_go_view(self.window)
		return bool(view)

	def run(self, prompt="go ", run="", fmt_save=True):
		view = gs.active_valid_go_view(self.window)
		if not view:
			gs.notice(DOMAIN, "this not a source.go view")
			return

		run = run.strip()
		p = Prompt(view, run == "")
		if run:
			p.on_done(run, fmt_save)
		else:
			p.panel = self.window.show_input_panel("GsShell", prompt, p.on_done, p.on_change, None)

def command_on_output(c, line):
	c.outq().put(line)

def command_on_done(c):
	pass

def fix_env(env):
	e = {}
	for k,v in env.items():
		e[k] = str(v)
	return e

def fix_shell_cmd(shell, cmd):
	if not gs.is_a(cmd, []):
		cmd = [cmd]

	if shell:
		sh = gs.setting('shell')
		cmd_str = ' '.join(cmd)
		if sh:
			shell = False
			cmd = []
			for v in sh:
				if v:
					cmd.append(str(v).replace('$CMD', cmd_str))
		else:
			cmd = [cmd_str]

	return (shell, [gs.astr(v) for v in cmd])

def run(cmd=[], shell=False, env={}, cwd=None, input=None):
	out = u""
	err = u""
	exc = None

	try:
		env = fix_env(env)
		shell, cmd = fix_shell_cmd(shell, cmd)
		p = gs.popen(cmd, shell=shell, stderr=subprocess.STDOUT, environ=env, cwd=cwd)
		if input is not None:
			input = gs.astr(input)
		out, _ = p.communicate(input=input)
		out = gs.ustr(out) if out else u''
	except Exception as ex:
		err = u'Error while running %s: %s' % (cmd, gs.traceback())
		exc = ex

	return (out, err, exc)

class CommandStdoutReader(threading.Thread):
	def __init__(self, c, stdout):
		super(CommandStdoutReader, self).__init__()
		self.daemon = True
		self.stdout = stdout
		self.c = c

	def run(self):
		try:
			while True:
				line = self.stdout.readline()

				if not line:
					self.c.close_stdout()
					break

				if not self.c.output_started:
					self.c.output_started = time.time()

				try:
					self.c.on_output(self.c, gs.ustr(line.rstrip('\r\n')))
				except Exception:
					gs.println(gs.traceback(DOMAIN))
		except Exception:
			gs.println(gs.traceback(DOMAIN))

class Command(threading.Thread):
	def __init__(self, cmd=[], shell=False, env={}, cwd=None):
		super(Command, self).__init__()
		self.daemon = True
		self.cancelled = False
		self.q = Queue.Queue()
		self.p = None
		self.x = None
		self.rcode = None
		self.started = 0
		self.output_started = 0
		self.ended = 0
		self.on_output = command_on_output
		self.on_done = command_on_done
		self.env = fix_env(env)
		self.shell, self.cmd = fix_shell_cmd(shell, cmd)
		self.message = str(self.cmd)
		self.cwd = cwd if cwd else None

	def outq(self):
		return self.q

	def process(self):
		return self.p

	def exception(self):
		return self.x

	def return_code(self):
		return self.rcode

	def consume_outq(self):
		l = []
		try:
			while True:
				l.append(self.q.get_nowait())
		except Queue.Empty:
			pass
		return l

	def poll(self):
		if self.p:
			return self.p.poll()
		return False

	def cancel(self):
		if self.poll() is None:
			try:
				os.killpg(self.p.pid, signal.SIGTERM)
			except Exception:
				self.p.terminate()

			time.sleep(0.100)
			if not self.completed():
				time.sleep(0.500)
				if not self.completed():
					try:
						os.killpg(self.p.pid, signal.SIGKILL)
					except Exception:
						try:
							self.p.kill()
						except Exception:
							pass
					self.close_stdout()

		discarded = 0
		try:
			while True:
				self.q.get_nowait()
				discarded += 1
		except Queue.Empty:
			pass

		return discarded

	def close_stdout(self):
		try:
			if self.p:
				self.p.stdout.close()
		except Exception:
			pass

	def completed(self):
		return self.return_code() is not None

	def run(self):
		self.started = time.time()
		tid = gs.begin(DOMAIN, self.message, set_status=False, cancel=self.cancel)
		try:
			try:
				self.p = gs.popen(self.cmd, shell=self.shell, stderr=subprocess.STDOUT,
					environ=self.env, cwd=self.cwd, bufsize=1)

				CommandStdoutReader(self, self.p.stdout).start()
			except Exception as ex:
				self.x = ex
			finally:
				self.rcode = self.p.wait()
		finally:
			gs.end(tid)
			self.ended = time.time()
			self.on_done(self)

class ViewCommand(Command):
	def __init__(self, cmd=[], shell=False, env={}, cwd='', view=None):
		self.view = view
		super(ViewCommand, self).__init__(cmd=cmd, shell=shell, env=env, cwd=cwd)

		if not self.cwd and view is not None:
			try:
				self.cwd = gs.basedir_or_cwd(view.file_name())
			except Exception:
				self.cwd = None

	def poll_output(self):
		l = []
		try:
			for i in range(500):
				l.append(self.q.get_nowait())
		except Queue.Empty:
			pass

		if l:
			self.do_insert(l)

		if not self.completed() or self.q.qsize() > 0:
			sublime.set_timeout(self.poll_output, 100)

	def do_insert(self, lines):
		if self.view is not None:
			edit = self.view.begin_edit()
			try:
				self.write_lines(self.view, edit, lines)
			finally:
				self.view.end_edit(edit)

	def write_lines(self, view, edit, lines):
		for ln in lines:
			try:
				view.insert(edit, view.size(), u'%s\n' % ln)
			except Exception:
				gs.println(gs.traceback(DOMAIN))
		view.show(view.line(view.size() - 1).begin())

	def on_done(self, c):
		ex = self.exception()
		if ex:
			self.on_output(c, 'Error: ' % ex)

		t = (max(0, c.ended - c.started), max(0, c.output_started - c.started))
		self.on_output(c, '[done: elapsed: %0.3fs, startup: %0.3fs]\n' % t)

	def cancel(self):
		discarded = super(ViewCommand, self).cancel()
		t = ((time.time() - self.started), discarded)
		self.on_output(self, ('\n[cancelled: elapsed: %0.3fs, discarded %d line(s)]\n' % t))

	def run(self):
		sublime.set_timeout(self.poll_output, 0)
		super(ViewCommand, self).run()
<commit_msg>* fix default arg<commit_after>import sublime
import sublime_plugin
import gscommon as gs
import re
import os
import httplib
import hashlib
import threading
import Queue
import traceback
import subprocess
import time
import signal

DOMAIN = "GsShell"
GO_RUN_PAT = re.compile(r'^go\s+(run|play)$', re.IGNORECASE)
GO_SHARE_PAT = re.compile(r'^go\s+share$', re.IGNORECASE)
GO_PLAY_PAT = re.compile(r'(\b)go\s+play(\b)', re.IGNORECASE)

class Prompt(object):
	def __init__(self, view, change_history=True):
		self.view = view
		self.panel = None
		self.subcommands = [
			'go run', 'go build', 'go clean', 'go fix',
			'go install', 'go test', 'go fmt', 'go vet', 'go tool',
			'go share', 'go play',
		]
		self.settings = sublime.load_settings('GoSublime-GsShell.sublime-settings')
		self.change_history = change_history

	def on_done(self, s, fmt_save=True):
		fn = self.view.file_name()
		win = self.view.window()
		if fn and win:
			basedir = os.path.dirname(fn)
			if fmt_save:
				for v in win.views():
					vfn = v.file_name()
					if vfn and v.is_dirty() and os.path.dirname(vfn) == basedir and vfn.endswith('.go'):
						v.run_command('gs_fmt_save')
		elif fmt_save:
			self.view.run_command('gs_fmt')

		# above we do some saves - thus creating a race so push this back to the end of the queue
		def cb(s):
			file_name = self.view.file_name() or ''
			s = GO_PLAY_PAT.sub(r'\1go run\2', s)
			s = s.strip()
			if s and s.lower() != "go" and self.change_history:
				hist = self.settings.get('cmd_hist')
				if not gs.is_a(hist, {}):
					hist = {}
				basedir = gs.basedir_or_cwd(file_name)
				hist[basedir] = [s] # todo: store a list of historical commands
				hst = {}
				for k in hist:
					# :|
					hst[gs.ustr(k)] = gs.ustr(hist[k])
				self.settings.set('cmd_hist', hst)
				sublime.save_settings('GoSublime-GsShell.sublime-settings')

			if GO_SHARE_PAT.match(s):
				s = ''
				host = "play.golang.org"
				warning = 'Are you sure you want to share this file. It will be public on %s' % host
				if not sublime.ok_cancel_dialog(warning):
					return

				try:
					c = httplib.HTTPConnection(host)
					src = gs.astr(self.view.substr(sublime.Region(0, self.view.size())))
					c.request('POST', '/share', src, {'User-Agent': 'GoSublime'})
					s = 'http://%s/p/%s' % (host, c.getresponse().read())
				except Exception as ex:
					s = 'Error: %s' % ex

				self.show_output(s, focus=True)
				return

			if GO_RUN_PAT.match(s):
				if not file_name:
					# todo: clean this up after the command runs
					err = ''
					tdir, _ = gs.temp_dir('play')
					file_name = hashlib.sha1(gs.view_fn(self.view) or 'a').hexdigest()
					file_name = os.path.join(tdir, ('%s.go' % file_name))
					try:
						with open(file_name, 'w') as f:
							src = gs.astr(self.view.substr(sublime.Region(0, self.view.size())))
							f.write(src)
					except Exception as ex:
						err = str(ex)

					if err:
						self.show_output('Error: %s' % err)
						return

				s = ['go', 'run', file_name]

			self.view.window().run_command("exec", { 'kill': True })
			if gs.is_a(s, []):
				use_shell = False
			else:
				use_shell = True
				s = [s]
			gs.println('running %s' % ' '.join(s))
			self.view.window().run_command("exec", {
				'shell': use_shell,
				'env': gs.env(),
				'cmd': s,
				'file_regex': '^(.+\.go):([0-9]+):(?:([0-9]+):)?\s*(.*)',
			})

		sublime.set_timeout(lambda: cb(s), 0)

	def show_output(self, s, focus=False):
		panel_name = DOMAIN+'-share'
		win = self.view.window()
		panel = win.get_output_panel(panel_name)
		edit = panel.begin_edit()
		try:
			panel.set_read_only(False)
			panel.sel().clear()
			panel.replace(edit, sublime.Region(0, panel.size()), s)
			panel.sel().add(sublime.Region(0, panel.size()))
			panel.set_read_only(True)
		finally:
			panel.end_edit(edit)
		print('%s output: %s' % (DOMAIN, s))
		self.view.window().run_command("show_panel", {"panel": "output.%s" % panel_name})
		if focus:
			sublime.set_timeout(lambda: win.focus_view(panel), 0)

	def on_change(self, s):
		if self.panel:
			size = self.view.size()
			if s.endswith('\t'):
				basedir = gs.basedir_or_cwd(self.view.file_name())
				lc = 'go '
				hist = self.settings.get('cmd_hist')
				if gs.is_a(hist, {}):
					hist = hist.get(basedir)
					if hist and gs.is_a(hist, []):
						lc = hist[-1]
				s = s.strip()
				if s and s not in ('', 'go'):
					l = []
					for i in self.subcommands:
						if i.startswith(s):
							l.append(i)
					if len(l) == 1:
						s = '%s ' % l[0]
				elif lc:
					s = '%s ' % lc
				edit = self.panel.begin_edit()
				try:
					self.panel.replace(edit, sublime.Region(0, self.panel.size()), s)
				finally:
					self.panel.end_edit(edit)

class GsShellCommand(sublime_plugin.WindowCommand):
	def is_enabled(self):
		view = gs.active_valid_go_view(self.window)
		return bool(view)

	def run(self, prompt="go ", run="", fmt_save=True):
		view = gs.active_valid_go_view(self.window)
		if not view:
			gs.notice(DOMAIN, "this not a source.go view")
			return

		run = run.strip()
		p = Prompt(view, run == "")
		if run:
			p.on_done(run, fmt_save)
		else:
			p.panel = self.window.show_input_panel("GsShell", prompt, p.on_done, p.on_change, None)

def command_on_output(c, line):
	c.outq().put(line)

def command_on_done(c):
	pass

def fix_env(env):
	e = {}
	for k,v in env.items():
		e[k] = str(v)
	return e

def fix_shell_cmd(shell, cmd):
	if not gs.is_a(cmd, []):
		cmd = [cmd]

	if shell:
		sh = gs.setting('shell')
		cmd_str = ' '.join(cmd)
		if sh:
			shell = False
			cmd = []
			for v in sh:
				if v:
					cmd.append(str(v).replace('$CMD', cmd_str))
		else:
			cmd = [cmd_str]

	return (shell, [gs.astr(v) for v in cmd])

def run(cmd=[], shell=False, env={}, cwd=None, input=None):
	out = u""
	err = u""
	exc = None

	try:
		env = fix_env(env)
		shell, cmd = fix_shell_cmd(shell, cmd)
		p = gs.popen(cmd, shell=shell, stderr=subprocess.STDOUT, environ=env, cwd=cwd)
		if input is not None:
			input = gs.astr(input)
		out, _ = p.communicate(input=input)
		out = gs.ustr(out) if out else u''
	except Exception as ex:
		err = u'Error while running %s: %s' % (cmd, gs.traceback())
		exc = ex

	return (out, err, exc)

class CommandStdoutReader(threading.Thread):
	def __init__(self, c, stdout):
		super(CommandStdoutReader, self).__init__()
		self.daemon = True
		self.stdout = stdout
		self.c = c

	def run(self):
		try:
			while True:
				line = self.stdout.readline()

				if not line:
					self.c.close_stdout()
					break

				if not self.c.output_started:
					self.c.output_started = time.time()

				try:
					self.c.on_output(self.c, gs.ustr(line.rstrip('\r\n')))
				except Exception:
					gs.println(gs.traceback(DOMAIN))
		except Exception:
			gs.println(gs.traceback(DOMAIN))

class Command(threading.Thread):
	def __init__(self, cmd=[], shell=False, env={}, cwd=None):
		super(Command, self).__init__()
		self.daemon = True
		self.cancelled = False
		self.q = Queue.Queue()
		self.p = None
		self.x = None
		self.rcode = None
		self.started = 0
		self.output_started = 0
		self.ended = 0
		self.on_output = command_on_output
		self.on_done = command_on_done
		self.env = fix_env(env)
		self.shell, self.cmd = fix_shell_cmd(shell, cmd)
		self.message = str(self.cmd)
		self.cwd = cwd if cwd else None

	def outq(self):
		return self.q

	def process(self):
		return self.p

	def exception(self):
		return self.x

	def return_code(self):
		return self.rcode

	def consume_outq(self):
		l = []
		try:
			while True:
				l.append(self.q.get_nowait())
		except Queue.Empty:
			pass
		return l

	def poll(self):
		if self.p:
			return self.p.poll()
		return False

	def cancel(self):
		if self.poll() is None:
			try:
				os.killpg(self.p.pid, signal.SIGTERM)
			except Exception:
				self.p.terminate()

			time.sleep(0.100)
			if not self.completed():
				time.sleep(0.500)
				if not self.completed():
					try:
						os.killpg(self.p.pid, signal.SIGKILL)
					except Exception:
						try:
							self.p.kill()
						except Exception:
							pass
					self.close_stdout()

		discarded = 0
		try:
			while True:
				self.q.get_nowait()
				discarded += 1
		except Queue.Empty:
			pass

		return discarded

	def close_stdout(self):
		try:
			if self.p:
				self.p.stdout.close()
		except Exception:
			pass

	def completed(self):
		return self.return_code() is not None

	def run(self):
		self.started = time.time()
		tid = gs.begin(DOMAIN, self.message, set_status=False, cancel=self.cancel)
		try:
			try:
				self.p = gs.popen(self.cmd, shell=self.shell, stderr=subprocess.STDOUT,
					environ=self.env, cwd=self.cwd, bufsize=1)

				CommandStdoutReader(self, self.p.stdout).start()
			except Exception as ex:
				self.x = ex
			finally:
				self.rcode = self.p.wait()
		finally:
			gs.end(tid)
			self.ended = time.time()
			self.on_done(self)

class ViewCommand(Command):
	def __init__(self, cmd=[], shell=False, env={}, cwd=None, view=None):
		self.view = view
		super(ViewCommand, self).__init__(cmd=cmd, shell=shell, env=env, cwd=cwd)

		if not self.cwd and view is not None:
			try:
				self.cwd = gs.basedir_or_cwd(view.file_name())
			except Exception:
				self.cwd = None

	def poll_output(self):
		l = []
		try:
			for i in range(500):
				l.append(self.q.get_nowait())
		except Queue.Empty:
			pass

		if l:
			self.do_insert(l)

		if not self.completed() or self.q.qsize() > 0:
			sublime.set_timeout(self.poll_output, 100)

	def do_insert(self, lines):
		if self.view is not None:
			edit = self.view.begin_edit()
			try:
				self.write_lines(self.view, edit, lines)
			finally:
				self.view.end_edit(edit)

	def write_lines(self, view, edit, lines):
		for ln in lines:
			try:
				view.insert(edit, view.size(), u'%s\n' % ln)
			except Exception:
				gs.println(gs.traceback(DOMAIN))
		view.show(view.line(view.size() - 1).begin())

	def on_done(self, c):
		ex = self.exception()
		if ex:
			self.on_output(c, 'Error: ' % ex)

		t = (max(0, c.ended - c.started), max(0, c.output_started - c.started))
		self.on_output(c, '[done: elapsed: %0.3fs, startup: %0.3fs]\n' % t)

	def cancel(self):
		discarded = super(ViewCommand, self).cancel()
		t = ((time.time() - self.started), discarded)
		self.on_output(self, ('\n[cancelled: elapsed: %0.3fs, discarded %d line(s)]\n' % t))

	def run(self):
		sublime.set_timeout(self.poll_output, 0)
		super(ViewCommand, self).run()
<|endoftext|>


<commit_before>import sys
import re
import os

from django.conf import settings
from logging import getLogger
from django.db.utils import IntegrityError

from astra.models import Admin, Account
from restclients.models.sws import Person
from restclients.models.canvas import CanvasUser
from restclients.exceptions import DataFailureException

from suds.client import Client
from suds.transport.http import HttpTransport
from suds import WebFault

from sis_provisioner.models import User
from sis_provisioner.loader import load_user
from sis_provisioner.dao.account import get_all_campuses, get_all_colleges,\
    get_departments_by_college
from sis_provisioner.dao.user import get_person_by_netid, user_fullname,\
    user_email
from sis_provisioner.dao.canvas import get_user_by_sis_id, create_user,\
    get_account_by_id, get_all_sub_accounts

import urllib2
import socket
import ssl
import httplib

class ASTRAException(Exception):
    pass

class HTTPSTransportV3(HttpTransport):
    def __init__(self, *args, **kwargs):
        HttpTransport.__init__(self, *args, **kwargs)

    def u2open(self, u2request):
        tm = self.options.timeout
        url = urllib2.build_opener(HTTPSClientAuthHandler())
        if self.u2ver() < 2.6:
            socket.setdefaulttimeout(tm)
            return url.open(u2request)
        else:
            return url.open(u2request, timeout=tm)

class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
    def __init__(self):
        urllib2.HTTPSHandler.__init__(self)

    def https_open(self, req):
        return self.do_open(HTTPSConnectionClientCertV3, req)

    def getConnection(self, host, timeout=300):
        return HTTPSConnectionClientCertV3(host)

class HTTPSConnectionClientCertV3(httplib.HTTPSConnection):
    def __init__(self, *args, **kwargs):
        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
        self.key_file = settings.ASTRA_KEY
        self.cert_file = settings.ASTRA_CERT

    def connect(self):
        sock = socket.create_connection((self.host, self.port), self.timeout)
        if self._tunnel_host:
            self.sock = sock
            self._tunnel()
        try:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_TLSv1)
        except ssl.SSLError, e:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_SSLv3)

class ASTRA():
    """Load admin table with ASTRA-defined administrators
    """
    def __init__(self, options={}):
        self._astra = Client(settings.ASTRA_WSDL,
                             transport=HTTPSTransportV3())
        # prepare to map spans of control to campus and college resource values
        self._campuses = get_all_campuses()
        self._colleges = get_all_colleges()
        self._pid = os.getpid()
        self._log = getLogger(__name__)
        self._re_non_academic_code = re.compile(r'^canvas_([0-9]+)$')
        self._canvas_ids = {}
        self._verbosity = int(options.get('verbosity', 0))

    def _request(self, methodName, params={}):
        port = 'AuthzProvider'
        try:
            result = self._astra.service[port][methodName](params)
            return result
        except WebFault, err:
            self._log.error(err)
        except:
            self._log.error('Other error: ' + str(sys.exc_info()[1]))

        return None

    def _getAuthz(self, authFilter):
        return self._request('GetAuthz', authFilter)

    def get_version(self):
        return self._request('GetVersion', {})

    def _add_admin(self, **kwargs):
        netid = kwargs['net_id']
        regid = kwargs['reg_id']
        self._log.info('ADD: %s is %s in %s' % (netid,
                                                kwargs['role'],
                                                kwargs['account_id']))

        try:
            User.objects.get(reg_id=regid)
        except User.DoesNotExist:
            try:
                person = get_person_by_netid(netid)

                self._log.info('Provisioning admin: %s (%s)' % (
                    person.uwnetid, person.uwregid))

                try:
                    user = get_user_by_sis_id(person.uwregid)
                except DataFailureException as err:
                    if err.status == 404:
                        user = create_user(CanvasUser(
                            name=user_fullname(person),
                            login_id=person.uwnetid,
                            sis_user_id=person.uwregid,
                            email=user_email(person)))

                load_user(person)

            except Exception, err:
                self._log.info('Skipped admin: %s (%s)' % (netid, err))
                return

        try:
            admin = Admin.objects.get(net_id=netid,
                                      reg_id=regid,
                                      account_id=kwargs['account_id'],
                                      canvas_id=kwargs['canvas_id'],
                                      role=kwargs['role'])
        except Admin.DoesNotExist:
            admin = Admin(net_id=netid,
                          reg_id=regid,
                          account_id=kwargs['account_id'],
                          canvas_id=kwargs['canvas_id'],
                          role=kwargs['role'],
                          queue_id=self._pid)

        admin.is_deleted = None
        admin.deleted_date = None
        admin.save()

    def _quote_id_level(self, level):
        return re.sub(r'[ :]', '-', level.lower())

    def _get_campus_from_code(self, code):
        for c in self._campuses:
            if c.label.lower() == code.lower():
                return c.label.lower()

        raise ASTRAException('Unknown Campus Code: %s' % code)

    def _get_college_from_code(self, campus, code):
        for college in self._colleges:
            if (campus.lower() == college.campus_label.lower() and
                    code.lower() == college.label.lower()):
                return college

        raise ASTRAException('Unknown College Code: %s' % code)

    def _valid_department_code(self, college, code):
        depts = get_departments_by_college(college)
        for dept in depts:
            if dept.label.lower() == code.lower():
                return code

        raise ASTRAException('Unknown Department Code: %s' % code)

    def _generate_sis_account_id(self, soc):
        id = []
        campus = None
        college = None

        if not isinstance(soc, list):
            raise ASTRAException('NO Span of Control')

        if soc[0]:
            if (soc[0]._type == 'CanvasNonAcademic' or
                    soc[0]._type == 'CanvasTestAccount'):
                try:
                    return (soc[0]._code,
                            self._re_non_academic_code.match(soc[0]._code).group(1))
                except Exception, err:
                    raise ASTRAException('Unknown non-academic code: %s %s' % (
                        soc[0]._code, err))
            elif soc[0]._type == 'SWSCampus':
                campus = self._get_campus_from_code(soc[0]._code)
                id.append(settings.SIS_IMPORT_ROOT_ACCOUNT_ID)
                id.append(self._quote_id_level(campus))
            else:
                raise ASTRAException('Unknown SOC type: %s %s' % (
                    soc[0]._type, soc[0]))

        if len(soc) > 1:
            if soc[1]._type == 'swscollege':
                if campus:
                    college = self._get_college_from_code(campus, soc[1]._code)
                    id.append(self._quote_id_level(college.name))
                else:
                    raise ASTRAException('College without campus: %s' % (
                        soc[1]._code))
            else:
                raise ASTRAException('Unknown second level SOC: %s' % (
                    soc[1]._type))

            if len(soc) > 2:
                if soc[2]._type == 'swsdepartment':
                    if (campus and college and
                            self._valid_department_code(college, soc[2]._code)):
                        id.append(self._quote_id_level(soc[2]._code))
                    else:
                        raise ASTRAException('Unknown third level SOC: %s' % (
                            soc[0]))

        sis_id = ':'.join(id)

        if sis_id not in self._canvas_ids:
            canvas_account = CanvasAccounts().get_account_by_sis_id(sis_id)
            self._canvas_ids[sis_id] = canvas_account.account_id

        return (sis_id, self._canvas_ids[sis_id])

    def load_all_admins(self, options={}):
        # loader running?
        queued = Admin.objects.queued()
        if len(queued):
            # look for pid matching queue_id, adjust gripe accordingly
            try:
                os.kill(queued[0].queue_id, 0)
                raise ASTRAException('loader already running %s' % (
                    queued[0].queue_id))
            except:
                override = options.get('override', 0)
                if override > 0 and override == queued[0].queue_id:
                    Admin.objects.dequeue(queue_id=override)
                    if len(Admin.objects.queued()):
                        raise ASTRAException('unable to override process %s' % (
                            override))
                else:
                    raise ASTRAException('loader blocked by process %s' % (
                        queued[0].queue_id))

        # query ASTRA
        authFilter = self._astra.factory.create('authFilter')
        authFilter.privilege._code = settings.ASTRA_APPLICATION
        authFilter.environment._code = settings.ASTRA_ENVIRONMENT
        authFilter.astraRole._code = 'User'

        authz = self._getAuthz(authFilter)
        if not authz:
            self._log.error(
                'ASTRA GetAuthz failed. Aborting Canvas admin update.')
            return

        # flag and mark all records deleted to catch ASTRA fallen
        Admin.objects.queue_all(queue_id=self._pid)

        # restore records with latest auths
        if 'authCollection' in authz and 'auth' in authz.authCollection:
            for auth in authz.authCollection.auth:
                try:
                    if auth.role._code not in settings.ASTRA_ROLE_MAPPING:
                        raise ASTRAException("Unknown Role Code: %s" % (
                            auth.role._code))
                    elif '_regid' not in auth.party:
                        raise ASTRAException("No regid in party: %s" % (
                            auth.party))

                    if 'spanOfControlCollection' in auth:
                        if ('spanOfControl' in auth.spanOfControlCollection and
                                isinstance(auth.spanOfControlCollection.spanOfControl, list)):
                            (account_id, canvas_id) = self._generate_sis_account_id(auth.spanOfControlCollection.spanOfControl)
                        else:
                            canvas_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
                            account_id = "canvas_%s" % canvas_id

                        self._add_admin(net_id=auth.party._uwNetid,
                                        reg_id=auth.party._regid,
                                        account_id=account_id,
                                        canvas_id=canvas_id,
                                        role=auth.role._code,
                                        is_deleted=None)
                    else:
                        raise ASTRAException("Missing required span of control: %s" % (auth.party))

                except ASTRAException, errstr:
                    self._log.error('%s\n AUTH: %s' % (errstr, auth))

        # log who fell from ASTRA
        for d in Admin.objects.get_deleted():
            self._log.info('REMOVE: %s as %s in %s' % (
                d.net_id, d.role, d.account_id))

        # tidy up
        Admin.objects.dequeue()

class Accounts():
    """Load account table with Canvas accounts
    """
    def __init__(self):
        self._re_sis = re.compile(r'^%s:(%s)(:[^:]+){0,3}$' % (
            settings.SIS_IMPORT_ROOT_ACCOUNT_ID,
            '|'.join([c.label.lower() for c in get_all_campuses()])))
        self._log = getLogger(__name__)

    def load_all_accounts(self):
        root_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
        accounts = [get_account(root_id)]
        accounts.extend(get_all_sub_accounts(root_id))

        Account.objects.all().update(is_deleted=True)

        for account in accounts:
            self.load_account(account)

    def load_account(self, account):
        sis_id = None
        account_type = Account.ADHOC_TYPE
        if account.account_id == int(settings.RESTCLIENTS_CANVAS_ACCOUNT_ID):
            account_type = Account.ROOT_TYPE
        elif account.sis_account_id is not None:
            sis_id = account.sis_account_id
            if self._re_sis.match(account.sis_account_id):
                account_type = Account.SDB_TYPE

        try:
            a = Account.objects.get(canvas_id=account.account_id)
            a.sis_id = sis_id
            a.account_name = account.name
            a.account_type = account_type
        except Account.DoesNotExist:
            a = Account(canvas_id=account.account_id,
                        sis_id=sis_id,
                        account_name=account.name,
                        account_type=account_type)

        a.is_deleted = None
        try:
            a.save()
        except IntegrityError, err:
            self._log.error(
                'ACCOUNT LOAD FAIL: canvas_id: %s, sis_id: %s, %s' % (
                    account.account_id, sis_id, err))
            raise

        return a
<commit_msg>use sis_provisioner canvas dao<commit_after>import sys
import re
import os

from django.conf import settings
from logging import getLogger
from django.db.utils import IntegrityError

from astra.models import Admin, Account
from restclients.models.sws import Person
from restclients.models.canvas import CanvasUser
from restclients.exceptions import DataFailureException

from suds.client import Client
from suds.transport.http import HttpTransport
from suds import WebFault

from sis_provisioner.models import User
from sis_provisioner.loader import load_user
from sis_provisioner.dao.account import get_all_campuses, get_all_colleges,\
    get_departments_by_college
from sis_provisioner.dao.user import get_person_by_netid, user_fullname,\
    user_email
from sis_provisioner.dao.canvas import get_user_by_sis_id, create_user,\
    get_account_by_id, get_account_by_sis_id, get_all_sub_accounts

import urllib2
import socket
import ssl
import httplib

class ASTRAException(Exception):
    pass

class HTTPSTransportV3(HttpTransport):
    def __init__(self, *args, **kwargs):
        HttpTransport.__init__(self, *args, **kwargs)

    def u2open(self, u2request):
        tm = self.options.timeout
        url = urllib2.build_opener(HTTPSClientAuthHandler())
        if self.u2ver() < 2.6:
            socket.setdefaulttimeout(tm)
            return url.open(u2request)
        else:
            return url.open(u2request, timeout=tm)

class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
    def __init__(self):
        urllib2.HTTPSHandler.__init__(self)

    def https_open(self, req):
        return self.do_open(HTTPSConnectionClientCertV3, req)

    def getConnection(self, host, timeout=300):
        return HTTPSConnectionClientCertV3(host)

class HTTPSConnectionClientCertV3(httplib.HTTPSConnection):
    def __init__(self, *args, **kwargs):
        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
        self.key_file = settings.ASTRA_KEY
        self.cert_file = settings.ASTRA_CERT

    def connect(self):
        sock = socket.create_connection((self.host, self.port), self.timeout)
        if self._tunnel_host:
            self.sock = sock
            self._tunnel()
        try:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_TLSv1)
        except ssl.SSLError, e:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_SSLv3)

class ASTRA():
    """Load admin table with ASTRA-defined administrators
    """
    def __init__(self, options={}):
        self._astra = Client(settings.ASTRA_WSDL,
                             transport=HTTPSTransportV3())
        # prepare to map spans of control to campus and college resource values
        self._campuses = get_all_campuses()
        self._colleges = get_all_colleges()
        self._pid = os.getpid()
        self._log = getLogger(__name__)
        self._re_non_academic_code = re.compile(r'^canvas_([0-9]+)$')
        self._canvas_ids = {}
        self._verbosity = int(options.get('verbosity', 0))

    def _request(self, methodName, params={}):
        port = 'AuthzProvider'
        try:
            result = self._astra.service[port][methodName](params)
            return result
        except WebFault, err:
            self._log.error(err)
        except:
            self._log.error('Other error: ' + str(sys.exc_info()[1]))

        return None

    def _getAuthz(self, authFilter):
        return self._request('GetAuthz', authFilter)

    def get_version(self):
        return self._request('GetVersion', {})

    def _add_admin(self, **kwargs):
        netid = kwargs['net_id']
        regid = kwargs['reg_id']
        self._log.info('ADD: %s is %s in %s' % (netid,
                                                kwargs['role'],
                                                kwargs['account_id']))

        try:
            User.objects.get(reg_id=regid)
        except User.DoesNotExist:
            try:
                person = get_person_by_netid(netid)

                self._log.info('Provisioning admin: %s (%s)' % (
                    person.uwnetid, person.uwregid))

                try:
                    user = get_user_by_sis_id(person.uwregid)
                except DataFailureException as err:
                    if err.status == 404:
                        user = create_user(CanvasUser(
                            name=user_fullname(person),
                            login_id=person.uwnetid,
                            sis_user_id=person.uwregid,
                            email=user_email(person)))

                load_user(person)

            except Exception, err:
                self._log.info('Skipped admin: %s (%s)' % (netid, err))
                return

        try:
            admin = Admin.objects.get(net_id=netid,
                                      reg_id=regid,
                                      account_id=kwargs['account_id'],
                                      canvas_id=kwargs['canvas_id'],
                                      role=kwargs['role'])
        except Admin.DoesNotExist:
            admin = Admin(net_id=netid,
                          reg_id=regid,
                          account_id=kwargs['account_id'],
                          canvas_id=kwargs['canvas_id'],
                          role=kwargs['role'],
                          queue_id=self._pid)

        admin.is_deleted = None
        admin.deleted_date = None
        admin.save()

    def _quote_id_level(self, level):
        return re.sub(r'[ :]', '-', level.lower())

    def _get_campus_from_code(self, code):
        for c in self._campuses:
            if c.label.lower() == code.lower():
                return c.label.lower()

        raise ASTRAException('Unknown Campus Code: %s' % code)

    def _get_college_from_code(self, campus, code):
        for college in self._colleges:
            if (campus.lower() == college.campus_label.lower() and
                    code.lower() == college.label.lower()):
                return college

        raise ASTRAException('Unknown College Code: %s' % code)

    def _valid_department_code(self, college, code):
        depts = get_departments_by_college(college)
        for dept in depts:
            if dept.label.lower() == code.lower():
                return code

        raise ASTRAException('Unknown Department Code: %s' % code)

    def _generate_sis_account_id(self, soc):
        id = []
        campus = None
        college = None

        if not isinstance(soc, list):
            raise ASTRAException('NO Span of Control')

        if soc[0]:
            if (soc[0]._type == 'CanvasNonAcademic' or
                    soc[0]._type == 'CanvasTestAccount'):
                try:
                    return (soc[0]._code,
                            self._re_non_academic_code.match(soc[0]._code).group(1))
                except Exception, err:
                    raise ASTRAException('Unknown non-academic code: %s %s' % (
                        soc[0]._code, err))
            elif soc[0]._type == 'SWSCampus':
                campus = self._get_campus_from_code(soc[0]._code)
                id.append(settings.SIS_IMPORT_ROOT_ACCOUNT_ID)
                id.append(self._quote_id_level(campus))
            else:
                raise ASTRAException('Unknown SOC type: %s %s' % (
                    soc[0]._type, soc[0]))

        if len(soc) > 1:
            if soc[1]._type == 'swscollege':
                if campus:
                    college = self._get_college_from_code(campus, soc[1]._code)
                    id.append(self._quote_id_level(college.name))
                else:
                    raise ASTRAException('College without campus: %s' % (
                        soc[1]._code))
            else:
                raise ASTRAException('Unknown second level SOC: %s' % (
                    soc[1]._type))

            if len(soc) > 2:
                if soc[2]._type == 'swsdepartment':
                    if (campus and college and
                            self._valid_department_code(college, soc[2]._code)):
                        id.append(self._quote_id_level(soc[2]._code))
                    else:
                        raise ASTRAException('Unknown third level SOC: %s' % (
                            soc[0]))

        sis_id = ':'.join(id)

        if sis_id not in self._canvas_ids:
            canvas_account = get_account_by_sis_id(sis_id)
            self._canvas_ids[sis_id] = canvas_account.account_id

        return (sis_id, self._canvas_ids[sis_id])

    def load_all_admins(self, options={}):
        # loader running?
        queued = Admin.objects.queued()
        if len(queued):
            # look for pid matching queue_id, adjust gripe accordingly
            try:
                os.kill(queued[0].queue_id, 0)
                raise ASTRAException('loader already running %s' % (
                    queued[0].queue_id))
            except:
                override = options.get('override', 0)
                if override > 0 and override == queued[0].queue_id:
                    Admin.objects.dequeue(queue_id=override)
                    if len(Admin.objects.queued()):
                        raise ASTRAException('unable to override process %s' % (
                            override))
                else:
                    raise ASTRAException('loader blocked by process %s' % (
                        queued[0].queue_id))

        # query ASTRA
        authFilter = self._astra.factory.create('authFilter')
        authFilter.privilege._code = settings.ASTRA_APPLICATION
        authFilter.environment._code = settings.ASTRA_ENVIRONMENT
        authFilter.astraRole._code = 'User'

        authz = self._getAuthz(authFilter)
        if not authz:
            self._log.error(
                'ASTRA GetAuthz failed. Aborting Canvas admin update.')
            return

        # flag and mark all records deleted to catch ASTRA fallen
        Admin.objects.queue_all(queue_id=self._pid)

        # restore records with latest auths
        if 'authCollection' in authz and 'auth' in authz.authCollection:
            for auth in authz.authCollection.auth:
                try:
                    if auth.role._code not in settings.ASTRA_ROLE_MAPPING:
                        raise ASTRAException("Unknown Role Code: %s" % (
                            auth.role._code))
                    elif '_regid' not in auth.party:
                        raise ASTRAException("No regid in party: %s" % (
                            auth.party))

                    if 'spanOfControlCollection' in auth:
                        if ('spanOfControl' in auth.spanOfControlCollection and
                                isinstance(auth.spanOfControlCollection.spanOfControl, list)):
                            (account_id, canvas_id) = self._generate_sis_account_id(auth.spanOfControlCollection.spanOfControl)
                        else:
                            canvas_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
                            account_id = "canvas_%s" % canvas_id

                        self._add_admin(net_id=auth.party._uwNetid,
                                        reg_id=auth.party._regid,
                                        account_id=account_id,
                                        canvas_id=canvas_id,
                                        role=auth.role._code,
                                        is_deleted=None)
                    else:
                        raise ASTRAException("Missing required span of control: %s" % (auth.party))

                except ASTRAException, errstr:
                    self._log.error('%s\n AUTH: %s' % (errstr, auth))

        # log who fell from ASTRA
        for d in Admin.objects.get_deleted():
            self._log.info('REMOVE: %s as %s in %s' % (
                d.net_id, d.role, d.account_id))

        # tidy up
        Admin.objects.dequeue()

class Accounts():
    """Load account table with Canvas accounts
    """
    def __init__(self):
        self._re_sis = re.compile(r'^%s:(%s)(:[^:]+){0,3}$' % (
            settings.SIS_IMPORT_ROOT_ACCOUNT_ID,
            '|'.join([c.label.lower() for c in get_all_campuses()])))
        self._log = getLogger(__name__)

    def load_all_accounts(self):
        root_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
        accounts = [get_account(root_id)]
        accounts.extend(get_all_sub_accounts(root_id))

        Account.objects.all().update(is_deleted=True)

        for account in accounts:
            self.load_account(account)

    def load_account(self, account):
        sis_id = None
        account_type = Account.ADHOC_TYPE
        if account.account_id == int(settings.RESTCLIENTS_CANVAS_ACCOUNT_ID):
            account_type = Account.ROOT_TYPE
        elif account.sis_account_id is not None:
            sis_id = account.sis_account_id
            if self._re_sis.match(account.sis_account_id):
                account_type = Account.SDB_TYPE

        try:
            a = Account.objects.get(canvas_id=account.account_id)
            a.sis_id = sis_id
            a.account_name = account.name
            a.account_type = account_type
        except Account.DoesNotExist:
            a = Account(canvas_id=account.account_id,
                        sis_id=sis_id,
                        account_name=account.name,
                        account_type=account_type)

        a.is_deleted = None
        try:
            a.save()
        except IntegrityError, err:
            self._log.error(
                'ACCOUNT LOAD FAIL: canvas_id: %s, sis_id: %s, %s' % (
                    account.account_id, sis_id, err))
            raise

        return a
<|endoftext|>