<commit_before>### tornado
import tornado.ioloop
import tornado.httpserver
import tornado.web
### self define from req import RequestHandler
from req import StaticFileHandler
from req import Service
### my app
import config
import myredis

### built-in module
import time
import signal
import logging
import momoko
import psycopg2.extras
from permission.permission import PermissionService

### service class
from service.user       import UserService
from service.problem    import ProblemService
from service.submission import SubmissionService
from service.testdata   import TestdataSerivce
from service.bulletin   import BulletinService
from service.execute    import ExecuteService
from service.contest    import ContestService
from service.verdict    import VerdictService
from service.group      import GroupService
from service.tag        import TagService
from service.school     import SchoolService
from service.verdictstring import VerdictStringService

### api class from api.user import ApiUserSignupHandler
from api.user           import ApiUserSigninHandler
from api.user           import ApiUserSignupHandler
from api.user           import ApiUserSignoutHandler
from api.user           import ApiUserResettokenHandler
from api.user           import ApiUsersHandler
from api.user           import ApiUserHandler
from api.user           import ApiUserGetInfoHandler
from api.user           import ApiUserGroupHandler
from api.bulletin       import ApiBulletinsHandler
from api.bulletin       import ApiBulletinHandler
from api.problem        import ApiProblemsHandler
from api.problem        import ApiProblemHandler
from api.problem        import ApiProblemExecuteHandler
from api.problem        import ApiProblemRejudgeHandler
from api.problem        import ApiProblemTagHandler
from api.submission     import ApiSubmissionsHandler
from api.submission     import ApiSubmissionHandler
from api.submission     import ApiSubmissionRejudgeHandler
from api.testdata       import ApiTestdatasHandler
from api.testdata       import ApiTestdataHandler
from api.contest        import ApiContestsHandler
from api.contest        import ApiContestHandler
from api.contest        import ApiContestProblemsHandler
from api.contest        import ApiContestSubmissionsHandler
#from api.contest        import ApiContestSubmissionHandler
from api.contest        import ApiContestScoreboardHandler
from api.execute        import ApiExecuteTypesHandler
from api.execute        import ApiExecuteTypesPriorityHandler
from api.execute        import ApiExecuteTypeHandler
from api.verdict        import ApiVerdictTypesHandler
from api.verdict        import ApiVerdictTypeHandler
from api.group          import ApiGroupHandler
from api.group          import ApiGroupsHandler
from api.group          import ApiGroupUserHandler
from api.group          import ApiGroupUserPowerHandler
from api.time           import ApiTimeHandler
from api.tag            import ApiTagsHandler
from api.tag            import ApiTagHandler

#from api.execute_type import ApiExecuteTypesHandler
#from api.execute_type import ApiExecuteTypeHandler

### web class
import web.modules

from web.err            import Web404Handler
from web.index          import WebIndexHandler
from web.bulletin       import WebBulletinsHandler
from web.bulletin       import WebBulletinHandler

from web.problem        import WebProblemsHandler
from web.problem        import WebProblemHandler
from web.problem        import WebProblemEditHandler

from web.submission     import WebSubmissionsHandler
from web.submission     import WebSubmissionHandler
from web.contest        import WebContestsHandler
from web.contest        import WebContestHandler
from web.contest        import WebContestEditHandler
from web.contest        import WebContestProblemHandler
from web.contest        import WebContestSubmissionsHandler
from web.contest        import WebContestSubmissionHandler
from web.contest        import WebContestScoreboardHandler

from web.execute        import WebExecuteTypesHandler
from web.execute        import WebExecuteTypeHandler
from web.verdict        import WebVerdictTypesHandler
from web.verdict        import WebVerdictTypeHandler

from web.group          import WebGroupHandler
from web.group          import WebGroupManageHandler
from web.group          import WebGroupsHandler

from web.about          import WebAboutHandler

#from web.execute_type import WebExecuteTypesHandler

from web.user           import WebUsersHandler
from web.user           import WebUserSignHandler
from web.user           import WebUserHandler
from web.user           import WebUserEditHandler
#from web.problem import WebProblemHandler
#from web.problem import WebProblemListHandler

### static file handler
from file.testdata import FileTestdataHandler

def sig_handler(sig, frame):
    print('Catch Stop Signal')
    tornado.ioloop.IOLoop.instance().add_callback(shutdown)

def shutdown():
    print('Server Stopping')
    srv.stop()
    io_loop = tornado.ioloop.IOLoop.instance()
    deadline = time.time() + config.MAX_WAIT_SECOND_BEFORE_SHUTDOWN

    def stop_loop():
        now = time.time()
        if now < deadline and (io_loop._callbacks or io_loop._timeouts):
            io_loop.add_timeout(now + 1, stop_loop)
        else:
            io_loop.stop()
            print('Server Stopped')
    stop_loop()

if __name__ == '__main__':
    print('Server Starting')
    '''
    logging.basicConfig(level=logging.DEBUG,
            format='[%(asctime)s]: %(message)s',
            datefmt='%a, %d %b %Y %H:%M:%S',
            filename=config.LOG_FILE_PATH,
            filemode='a')
    '''
    db = momoko.Pool(
            dsn = 'dbname=%s user=%s password=%s host=%s port=%s'%(config.DBNAME, config.DBUSER, config.DBPASSWORD, config.DBHOST, config.DBPORT),
            size = config.DBMIN_SIZE,
            max_size = config.DBMAX_SIZE,
            ioloop = tornado.ioloop.IOLoop.instance(),
            setsession = ("SET TIME ZONE +8",),
            raise_connect_errors=False,
            cursor_factory = psycopg2.extras.RealDictCursor
            )
    future = db.connect()
    tornado.ioloop.IOLoop.instance().add_future(future, lambda f: tornado.ioloop.IOLoop.instance().stop())
    tornado.ioloop.IOLoop.instance().start()
    rs = myredis.MyRedis(db=1)
    rs.flushdb()
    ui_modules = {
            "Pagination": web.modules.Pagination,
            }
    app = tornado.web.Application([
        ('/',                                                           WebIndexHandler),
        ('/api/users/',                                                 ApiUsersHandler),
        ('/api/users/getinfo/',                                         ApiUserGetInfoHandler),
        ('/api/users/(\d+)/',                                           ApiUserHandler),
        ('/api/users/(\d+)/groups/(\d+)/(problems)/',                   ApiUserGroupHandler),
        ('/api/users/signin/',                                          ApiUserSigninHandler),
        ('/api/users/signup/',                                          ApiUserSignupHandler),
        ('/api/users/signout/',                                         ApiUserSignoutHandler),
        ('/api/users/resettoken/',                                      ApiUserResettokenHandler),

        ('/api/groups/',                                                ApiGroupsHandler),
        ('/api/groups/\d+/',                                            ApiGroupHandler),
        ('/api/groups/\d+/(\d+)/',                                      ApiGroupUserHandler),
        ('/api/groups/\d+/(\d+)/power/',                                ApiGroupUserPowerHandler),
        ('/api/groups/\d+/bulletins/',                                  ApiBulletinsHandler),
        ('/api/groups/\d+/bulletins/(\d+)/',                            ApiBulletinHandler),

        ('/api/groups/\d+/problems/',                                   ApiProblemsHandler),
        ('/api/groups/\d+/problems/(\d+)/',                             ApiProblemHandler),
        ('/api/groups/\d+/problems/(\d+)/execute/',                     ApiProblemExecuteHandler),
        ('/api/groups/\d+/problems/(\d+)/rejudge/',                     ApiProblemRejudgeHandler),
        ('/api/groups/\d+/problems/(\d+)/tag/',                         ApiProblemTagHandler),
        # ('/api/groups/\d+/problems/(\d+)/(\w*)/',                        ApiProblemHandler),
        ### /api/groups/\d+/problems/(\d+)/(\w*)/
        ### /api/groups/\d+/problems/(\d+)/(tag)/
        ### /api/groups/\d+/problems/(\d+)/(testdata)/
        ('/api/groups/\d+/problems/(\d+)/(\w*)/(\d+)/',                 ApiProblemHandler),
        ### /api/groups/\d+/problems/(\d+)/(testdata)/(\d+)/'
        ('/api/groups/\d+/testdata/',                                   ApiTestdatasHandler),
        ('/api/groups/\d+/testdata/(\d+)/',                             ApiTestdataHandler),
        ('/api/groups/\d+/submissions/',                                ApiSubmissionsHandler),
        ('/api/groups/\d+/submissions/(\d+)/',                          ApiSubmissionHandler),
        ('/api/groups/\d+/submissions/(\d+)/rejudge/',                  ApiSubmissionRejudgeHandler),
        ('/api/groups/\d+/submissions/(\d+)/(\w*)/',                    ApiSubmissionHandler),

        ('/api/groups/\d+/contests/',                                   ApiContestsHandler),
        ### TODO
        ('/api/groups/\d+/contests/(\d+)/',                             ApiContestHandler),
        ('/api/groups/\d+/contests/(\d+)/problems/',                    ApiContestProblemsHandler),
        ('/api/groups/\d+/contests/(\d+)/submissions/',                 ApiContestSubmissionsHandler),
        ('/api/groups/\d+/contests/(\d+)/scoreboard/',                  ApiContestScoreboardHandler),
        ('/api/groups/\d+/contests/(\d+)/(\w+)/',                       ApiContestHandler),
        ('/api/time/',                                                  ApiTimeHandler),

        ('/api/executes/',                                              ApiExecuteTypesHandler),
        ('/api/executes/priority/',                                     ApiExecuteTypesPriorityHandler),
        ('/api/executes/(\d+)/',                                        ApiExecuteTypeHandler),
        ('/api/verdicts/',                                              ApiVerdictTypesHandler),
        ('/api/verdicts/(\d+)/',                                        ApiVerdictTypeHandler),
        ('/api/tags/',                                                  ApiTagsHandler),
        ('/api/tags/(\d+)/',                                            ApiTagHandler),

        ('/groups/',                                                    WebGroupsHandler),

        ('/groups/(\d+)/',                                              WebGroupHandler),
        ('/groups/(\d+)/manage/(\w*/)?',                                WebGroupManageHandler),
        ('/groups/\d+/bulletins/',                                      WebBulletinsHandler),
        ('/groups/\d+/bulletins/(\d+)/(\w*)/',                          WebBulletinHandler),

        ('/groups/\d+/problems/',                                       WebProblemsHandler),
        ('/groups/\d+/problems/(\d+)/',                                 WebProblemHandler),
        ('/groups/\d+/problems/(\d+)/(\w*)/',                           WebProblemHandler),
        ('/groups/\d+/problems/(\d+)/(\w*)/edit/',                      WebProblemEditHandler),
        ### /groups/\d+/problems/\d+/basic/edit/
        ### /groups/\d+/problems/\d+/tag/edit/
        ### /groups/\d+/problems/\d+/execute/edit/
        ### /groups/\d+/problems/\d+/testdata/edit/
        ### /groups/\d+/problems/\d+/submit/

        ('/groups/\d+/submissions/',                                    WebSubmissionsHandler),
        ('/groups/\d+/submissions/(\d+)/',                              WebSubmissionHandler),

        ('/groups/\d+/contests/',                                       WebContestsHandler),
        ('/groups/\d+/contests/(\d+)/',                                 WebContestHandler),
        ### TODO
        ('/groups/\d+/contests/(\d+)/edit/',                            WebContestEditHandler),
        ('/groups/\d+/contests/(\d+)/problems/(\d+)/',                  WebContestProblemHandler),
        ('/groups/\d+/contests/(\d+)/problems/(\d+)/(\w*)/',            WebContestProblemHandler),
        ('/groups/\d+/contests/(\d+)/submissions/',                     WebContestSubmissionsHandler),
        ('/groups/\d+/contests/(\d+)/submissions/(\d+)/',               WebContestSubmissionHandler),
        ('/groups/\d+/contests/(\d+)/scoreboard/',                      WebContestScoreboardHandler),

        #('/groups/\d+/contests/(\d+)/rank/',                             WebContestRankHandler),

        ('/executes/',                                                  WebExecuteTypesHandler),
        ('/executes/(\d+)/',                                            WebExecuteTypeHandler),
        ('/executes/(\d+)/(\w*)/',                                      WebExecuteTypeHandler),
        ('/verdicts/',                                                  WebVerdictTypesHandler),
        ('/verdicts/(\d+)/',                                            WebVerdictTypeHandler),
        ('/verdicts/(\d+)/(\w*)/',                                      WebVerdictTypeHandler),

        ### user list only admin
        ('/users/',                                                     WebUsersHandler),
        ('/users/(\d+)/',                                               WebUserHandler),
        ('/users/(sign\w*)/',                                           WebUserSignHandler),
        ('/users/(\d+)/',                                               WebUserHandler),
        ('/users/(\d+)/edit/',                                          WebUserEditHandler),

        ('/about/',                                                     WebAboutHandler),
        ('/resource/testdata/(.*)', FileTestdataHandler, {'path': '%s/data/testdata'%(config.DATAROOT)}),
        ('/resource/(.*)', tornado.web.StaticFileHandler, {'path': '%s/data'%(config.DATAROOT)}),
        ('/asset/(.*)', tornado.web.StaticFileHandler, {'path': '../http'}),
        ('/(google4e9e359eaf9accab.html)', tornado.web.StaticFileHandler, {'path': '../http'}),
        ('/.*',                                                         Web404Handler),
        ],  cookie_secret = config.COOKIE_SECRET,
            compress_response = True,
            debug = config.DEBUG,
            autoescape =    'xhtml_escape',
            ui_modules =    ui_modules,
            xheaders=True,)
    global srv
    srv = tornado.httpserver.HTTPServer(app)
    srv.listen(config.PORT)
    Service.User =          UserService(db, rs)
    Service.Problem =       ProblemService(db, rs)
    Service.Submission =    SubmissionService(db, rs)
    Service.Testdata =      TestdataSerivce(db, rs)
    Service.Bulletin =      BulletinService(db, rs)
    Service.Execute =       ExecuteService(db, rs)
    Service.Contest =       ContestService(db, rs)
    Service.Verdict =       VerdictService(db, rs)
    Service.Group =         GroupService(db, rs)
    Service.Tags =          TagService(db, rs)
    Service.School =        SchoolService(db, rs)
    Service.VerdictString = VerdictStringService(db, rs)
    Service.Permission =    PermissionService
    signal.signal(signal.SIGTERM, sig_handler)
    signal.signal(signal.SIGINT, sig_handler)
    print('Server Started')
    tornado.ioloop.IOLoop().instance().start()
<commit_msg>support multiprocess<commit_after>### tornado
import tornado.ioloop
import tornado.httpserver
import tornado.web
import tornado.netutil
import tornado.process
### self define from req import RequestHandler
from req import StaticFileHandler
from req import Service
### my app
import config
import myredis

### built-in module
import time
import signal
import logging
import momoko
import psycopg2.extras
from permission.permission import PermissionService

### service class
from service.user       import UserService
from service.problem    import ProblemService
from service.submission import SubmissionService
from service.testdata   import TestdataSerivce
from service.bulletin   import BulletinService
from service.execute    import ExecuteService
from service.contest    import ContestService
from service.verdict    import VerdictService
from service.group      import GroupService
from service.tag        import TagService
from service.school     import SchoolService
from service.verdictstring import VerdictStringService

### api class from api.user import ApiUserSignupHandler
from api.user           import ApiUserSigninHandler
from api.user           import ApiUserSignupHandler
from api.user           import ApiUserSignoutHandler
from api.user           import ApiUserResettokenHandler
from api.user           import ApiUsersHandler
from api.user           import ApiUserHandler
from api.user           import ApiUserGetInfoHandler
from api.user           import ApiUserGroupHandler
from api.bulletin       import ApiBulletinsHandler
from api.bulletin       import ApiBulletinHandler
from api.problem        import ApiProblemsHandler
from api.problem        import ApiProblemHandler
from api.problem        import ApiProblemExecuteHandler
from api.problem        import ApiProblemRejudgeHandler
from api.problem        import ApiProblemTagHandler
from api.submission     import ApiSubmissionsHandler
from api.submission     import ApiSubmissionHandler
from api.submission     import ApiSubmissionRejudgeHandler
from api.testdata       import ApiTestdatasHandler
from api.testdata       import ApiTestdataHandler
from api.contest        import ApiContestsHandler
from api.contest        import ApiContestHandler
from api.contest        import ApiContestProblemsHandler
from api.contest        import ApiContestSubmissionsHandler
#from api.contest        import ApiContestSubmissionHandler
from api.contest        import ApiContestScoreboardHandler
from api.execute        import ApiExecuteTypesHandler
from api.execute        import ApiExecuteTypesPriorityHandler
from api.execute        import ApiExecuteTypeHandler
from api.verdict        import ApiVerdictTypesHandler
from api.verdict        import ApiVerdictTypeHandler
from api.group          import ApiGroupHandler
from api.group          import ApiGroupsHandler
from api.group          import ApiGroupUserHandler
from api.group          import ApiGroupUserPowerHandler
from api.time           import ApiTimeHandler
from api.tag            import ApiTagsHandler
from api.tag            import ApiTagHandler

#from api.execute_type import ApiExecuteTypesHandler
#from api.execute_type import ApiExecuteTypeHandler

### web class
import web.modules

from web.err            import Web404Handler
from web.index          import WebIndexHandler
from web.bulletin       import WebBulletinsHandler
from web.bulletin       import WebBulletinHandler

from web.problem        import WebProblemsHandler
from web.problem        import WebProblemHandler
from web.problem        import WebProblemEditHandler

from web.submission     import WebSubmissionsHandler
from web.submission     import WebSubmissionHandler
from web.contest        import WebContestsHandler
from web.contest        import WebContestHandler
from web.contest        import WebContestEditHandler
from web.contest        import WebContestProblemHandler
from web.contest        import WebContestSubmissionsHandler
from web.contest        import WebContestSubmissionHandler
from web.contest        import WebContestScoreboardHandler

from web.execute        import WebExecuteTypesHandler
from web.execute        import WebExecuteTypeHandler
from web.verdict        import WebVerdictTypesHandler
from web.verdict        import WebVerdictTypeHandler

from web.group          import WebGroupHandler
from web.group          import WebGroupManageHandler
from web.group          import WebGroupsHandler

from web.about          import WebAboutHandler

#from web.execute_type import WebExecuteTypesHandler

from web.user           import WebUsersHandler
from web.user           import WebUserSignHandler
from web.user           import WebUserHandler
from web.user           import WebUserEditHandler
#from web.problem import WebProblemHandler
#from web.problem import WebProblemListHandler

### static file handler
from file.testdata import FileTestdataHandler

def sig_handler(sig, frame):
    print('Catch Stop Signal')
    tornado.ioloop.IOLoop.instance().add_callback(shutdown)

def shutdown():
    print('Server Stopping')
    srv.stop()
    io_loop = tornado.ioloop.IOLoop.instance()
    deadline = time.time() + config.MAX_WAIT_SECOND_BEFORE_SHUTDOWN

    def stop_loop():
        now = time.time()
        if now < deadline and (io_loop._callbacks or io_loop._timeouts):
            io_loop.add_timeout(now + 1, stop_loop)
        else:
            io_loop.stop()
            print('Server Stopped')
    stop_loop()

if __name__ == '__main__':
    print('Server Starting')
    if not config.DEBUG:
        sock = tornado.netutil.bind_sockets(config.PORT)
        tornado.process.fork_processes(0)
    '''
    logging.basicConfig(level=logging.DEBUG,
            format='[%(asctime)s]: %(message)s',
            datefmt='%a, %d %b %Y %H:%M:%S',
            filename=config.LOG_FILE_PATH,
            filemode='a')
    '''
    db = momoko.Pool(
            dsn = 'dbname=%s user=%s password=%s host=%s port=%s'%(config.DBNAME, config.DBUSER, config.DBPASSWORD, config.DBHOST, config.DBPORT),
            size = config.DBMIN_SIZE,
            max_size = config.DBMAX_SIZE,
            ioloop = tornado.ioloop.IOLoop.instance(),
            setsession = ("SET TIME ZONE +8",),
            raise_connect_errors=False,
            cursor_factory = psycopg2.extras.RealDictCursor
            )
    future = db.connect()
    tornado.ioloop.IOLoop.instance().add_future(future, lambda f: tornado.ioloop.IOLoop.instance().stop())
    tornado.ioloop.IOLoop.instance().start()
    rs = myredis.MyRedis(db=1)
    rs.flushdb()
    ui_modules = {
            "Pagination": web.modules.Pagination,
            }
    app = tornado.web.Application([
        ('/',                                                           WebIndexHandler),
        ('/api/users/',                                                 ApiUsersHandler),
        ('/api/users/getinfo/',                                         ApiUserGetInfoHandler),
        ('/api/users/(\d+)/',                                           ApiUserHandler),
        ('/api/users/(\d+)/groups/(\d+)/(problems)/',                   ApiUserGroupHandler),
        ('/api/users/signin/',                                          ApiUserSigninHandler),
        ('/api/users/signup/',                                          ApiUserSignupHandler),
        ('/api/users/signout/',                                         ApiUserSignoutHandler),
        ('/api/users/resettoken/',                                      ApiUserResettokenHandler),

        ('/api/groups/',                                                ApiGroupsHandler),
        ('/api/groups/\d+/',                                            ApiGroupHandler),
        ('/api/groups/\d+/(\d+)/',                                      ApiGroupUserHandler),
        ('/api/groups/\d+/(\d+)/power/',                                ApiGroupUserPowerHandler),
        ('/api/groups/\d+/bulletins/',                                  ApiBulletinsHandler),
        ('/api/groups/\d+/bulletins/(\d+)/',                            ApiBulletinHandler),

        ('/api/groups/\d+/problems/',                                   ApiProblemsHandler),
        ('/api/groups/\d+/problems/(\d+)/',                             ApiProblemHandler),
        ('/api/groups/\d+/problems/(\d+)/execute/',                     ApiProblemExecuteHandler),
        ('/api/groups/\d+/problems/(\d+)/rejudge/',                     ApiProblemRejudgeHandler),
        ('/api/groups/\d+/problems/(\d+)/tag/',                         ApiProblemTagHandler),
        # ('/api/groups/\d+/problems/(\d+)/(\w*)/',                        ApiProblemHandler),
        ### /api/groups/\d+/problems/(\d+)/(\w*)/
        ### /api/groups/\d+/problems/(\d+)/(tag)/
        ### /api/groups/\d+/problems/(\d+)/(testdata)/
        ('/api/groups/\d+/problems/(\d+)/(\w*)/(\d+)/',                 ApiProblemHandler),
        ### /api/groups/\d+/problems/(\d+)/(testdata)/(\d+)/'
        ('/api/groups/\d+/testdata/',                                   ApiTestdatasHandler),
        ('/api/groups/\d+/testdata/(\d+)/',                             ApiTestdataHandler),
        ('/api/groups/\d+/submissions/',                                ApiSubmissionsHandler),
        ('/api/groups/\d+/submissions/(\d+)/',                          ApiSubmissionHandler),
        ('/api/groups/\d+/submissions/(\d+)/rejudge/',                  ApiSubmissionRejudgeHandler),
        ('/api/groups/\d+/submissions/(\d+)/(\w*)/',                    ApiSubmissionHandler),

        ('/api/groups/\d+/contests/',                                   ApiContestsHandler),
        ### TODO
        ('/api/groups/\d+/contests/(\d+)/',                             ApiContestHandler),
        ('/api/groups/\d+/contests/(\d+)/problems/',                    ApiContestProblemsHandler),
        ('/api/groups/\d+/contests/(\d+)/submissions/',                 ApiContestSubmissionsHandler),
        ('/api/groups/\d+/contests/(\d+)/scoreboard/',                  ApiContestScoreboardHandler),
        ('/api/groups/\d+/contests/(\d+)/(\w+)/',                       ApiContestHandler),
        ('/api/time/',                                                  ApiTimeHandler),

        ('/api/executes/',                                              ApiExecuteTypesHandler),
        ('/api/executes/priority/',                                     ApiExecuteTypesPriorityHandler),
        ('/api/executes/(\d+)/',                                        ApiExecuteTypeHandler),
        ('/api/verdicts/',                                              ApiVerdictTypesHandler),
        ('/api/verdicts/(\d+)/',                                        ApiVerdictTypeHandler),
        ('/api/tags/',                                                  ApiTagsHandler),
        ('/api/tags/(\d+)/',                                            ApiTagHandler),

        ('/groups/',                                                    WebGroupsHandler),

        ('/groups/(\d+)/',                                              WebGroupHandler),
        ('/groups/(\d+)/manage/(\w*/)?',                                WebGroupManageHandler),
        ('/groups/\d+/bulletins/',                                      WebBulletinsHandler),
        ('/groups/\d+/bulletins/(\d+)/(\w*)/',                          WebBulletinHandler),

        ('/groups/\d+/problems/',                                       WebProblemsHandler),
        ('/groups/\d+/problems/(\d+)/',                                 WebProblemHandler),
        ('/groups/\d+/problems/(\d+)/(\w*)/',                           WebProblemHandler),
        ('/groups/\d+/problems/(\d+)/(\w*)/edit/',                      WebProblemEditHandler),
        ### /groups/\d+/problems/\d+/basic/edit/
        ### /groups/\d+/problems/\d+/tag/edit/
        ### /groups/\d+/problems/\d+/execute/edit/
        ### /groups/\d+/problems/\d+/testdata/edit/
        ### /groups/\d+/problems/\d+/submit/

        ('/groups/\d+/submissions/',                                    WebSubmissionsHandler),
        ('/groups/\d+/submissions/(\d+)/',                              WebSubmissionHandler),

        ('/groups/\d+/contests/',                                       WebContestsHandler),
        ('/groups/\d+/contests/(\d+)/',                                 WebContestHandler),
        ### TODO
        ('/groups/\d+/contests/(\d+)/edit/',                            WebContestEditHandler),
        ('/groups/\d+/contests/(\d+)/problems/(\d+)/',                  WebContestProblemHandler),
        ('/groups/\d+/contests/(\d+)/problems/(\d+)/(\w*)/',            WebContestProblemHandler),
        ('/groups/\d+/contests/(\d+)/submissions/',                     WebContestSubmissionsHandler),
        ('/groups/\d+/contests/(\d+)/submissions/(\d+)/',               WebContestSubmissionHandler),
        ('/groups/\d+/contests/(\d+)/scoreboard/',                      WebContestScoreboardHandler),

        #('/groups/\d+/contests/(\d+)/rank/',                             WebContestRankHandler),

        ('/executes/',                                                  WebExecuteTypesHandler),
        ('/executes/(\d+)/',                                            WebExecuteTypeHandler),
        ('/executes/(\d+)/(\w*)/',                                      WebExecuteTypeHandler),
        ('/verdicts/',                                                  WebVerdictTypesHandler),
        ('/verdicts/(\d+)/',                                            WebVerdictTypeHandler),
        ('/verdicts/(\d+)/(\w*)/',                                      WebVerdictTypeHandler),

        ### user list only admin
        ('/users/',                                                     WebUsersHandler),
        ('/users/(\d+)/',                                               WebUserHandler),
        ('/users/(sign\w*)/',                                           WebUserSignHandler),
        ('/users/(\d+)/',                                               WebUserHandler),
        ('/users/(\d+)/edit/',                                          WebUserEditHandler),

        ('/about/',                                                     WebAboutHandler),
        ('/resource/testdata/(.*)', FileTestdataHandler, {'path': '%s/data/testdata'%(config.DATAROOT)}),
        ('/resource/(.*)', tornado.web.StaticFileHandler, {'path': '%s/data'%(config.DATAROOT)}),
        ('/asset/(.*)', tornado.web.StaticFileHandler, {'path': '../http'}),
        ('/(google4e9e359eaf9accab.html)', tornado.web.StaticFileHandler, {'path': '../http'}),
        ('/.*',                                                         Web404Handler),
        ],  cookie_secret = config.COOKIE_SECRET,
            compress_response = True,
            debug = config.DEBUG,
            autoescape =    'xhtml_escape',
            ui_modules =    ui_modules,
            xheaders=True,)
    global srv
    srv = tornado.httpserver.HTTPServer(app)
    if not config.DEBUG: srv.add_sockets(sock)
    else: srv.listen(config.PORT)
    Service.User =          UserService(db, rs)
    Service.Problem =       ProblemService(db, rs)
    Service.Submission =    SubmissionService(db, rs)
    Service.Testdata =      TestdataSerivce(db, rs)
    Service.Bulletin =      BulletinService(db, rs)
    Service.Execute =       ExecuteService(db, rs)
    Service.Contest =       ContestService(db, rs)
    Service.Verdict =       VerdictService(db, rs)
    Service.Group =         GroupService(db, rs)
    Service.Tags =          TagService(db, rs)
    Service.School =        SchoolService(db, rs)
    Service.VerdictString = VerdictStringService(db, rs)
    Service.Permission =    PermissionService
    signal.signal(signal.SIGTERM, sig_handler)
    signal.signal(signal.SIGINT, sig_handler)
    print('Server Started')
    tornado.ioloop.IOLoop().instance().start()
<|endoftext|>


<commit_before># Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

ALLOWED_HOSTS = ['www.oeconsortium.org']

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'haystack',
    'email_obfuscator',

    'directory',
    'web',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'directory.urls'
WSGI_APPLICATION = 'directory.wsgi.application'

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = False

STATIC_URL = '/static/'

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 100

from .localsettings import *<commit_msg>enable error reporting<commit_after># Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

ADMINS = (
    ('Jure Cuhalev', 'jure@oeconsortium.org'),
)

MANAGERS = ADMINS

ALLOWED_HOSTS = ['www.oeconsortium.org']

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'haystack',
    'email_obfuscator',

    'directory',
    'web',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'directory.urls'
WSGI_APPLICATION = 'directory.wsgi.application'

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = False

STATIC_URL = '/static/'

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 100

from .localsettings import *<|endoftext|>


<commit_before>"""Place of record for the package version"""

__version__ = "2.0.20"
__rpm_version__ = "WILL_BE_SET_BY_RPM_BUILD"
__git_hash__ = "WILL_BE_SET_BY_EGG_BUILD"
<commit_msg>Bumped asiaq version (again)<commit_after>"""Place of record for the package version"""

__version__ = "2.0.21"
__rpm_version__ = "WILL_BE_SET_BY_RPM_BUILD"
__git_hash__ = "WILL_BE_SET_BY_EGG_BUILD"
<|endoftext|>


<commit_before>from django.db import models
from django.core.validators import MinValueValidator

class Company(models.Model):
    """Describes a company/organization participating or interested in D-Dagen"""

    # filled in by company
    AREA_OF_BUSINESS_CHOICES = (
        ("finance", "Finance"),
        ("appdev", "App Development"),
        ("consulting", "Consulting"),
        ("games", "Games"),
        ("comm", "Communication")
    )
    company_name = models.CharField(max_length=100)
    description = models.CharField(max_length=1100)
    logotype = models.FileField()
    area_of_business = models.CharField(choices=AREA_OF_BUSINESS_CHOICES, default="Pick one", max_length=100)
    employees = models.IntegerField(validators=[MinValueValidator(1)])
    first_time_at_fair = models.BooleanField(default=True)
    billing_address = models.CharField(max_length=200)

    # for internal use only
    PRIORITY_CHOICES = (
        ("low", "LOW"),
        ("mid", "MID"),
        ("high","HIGH")
    )
    comment = models.CharField(max_length=500)
    priority = models.CharField(choices=PRIORITY_CHOICES, default=PRIORITY_CHOICES[0], max_length=100)
<commit_msg>improved field name and created string representation for Company model<commit_after>from django.db import models
from django.core.validators import MinValueValidator

class Company(models.Model):
    """Describes a company/organization participating or interested in D-Dagen"""

    def __str__(self):
        return self.name + ": " + area_of_business

    # filled in by company
    AREA_OF_BUSINESS_CHOICES = (
        ("finance", "Finance"),
        ("appdev", "App Development"),
        ("consulting", "Consulting"),
        ("games", "Games"),
        ("comm", "Communication")
    )
    name = models.CharField(max_length=100)
    description = models.CharField(max_length=1100)
    logotype = models.FileField()
    area_of_business = models.CharField(choices=AREA_OF_BUSINESS_CHOICES, default="Pick one", max_length=100)
    employees = models.IntegerField(validators=[MinValueValidator(1)])
    first_time_at_fair = models.BooleanField(default=True)
    billing_address = models.CharField(max_length=200)

    # for internal use only
    PRIORITY_CHOICES = (
        ("low", "LOW"),
        ("mid", "MID"),
        ("high","HIGH")
    )
    comment = models.CharField(max_length=500)
    priority = models.CharField(choices=PRIORITY_CHOICES, default=PRIORITY_CHOICES[0], max_length=100)
<|endoftext|>


<commit_before>import datetime
import calendar
from json import JSONEncoder, dumps

from flask import Response

from inbox.server.models.tables.base import (
    Message, SharedFolder, User, Account, Part,
    Contact, Thread, Namespace, Block)

# Flask's jsonify() doesn't handle datetimes or
# json arrays as primary objects
class APIEncoder(JSONEncoder):

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return calendar.timegm(obj.utctimetuple())

        elif isinstance(obj, Namespace):
            return {
                'id':  obj.public_id,
                'object': 'namepace',
                'ns':  obj.public_id,

                # Account specific
                'account':  obj.account.public_id,
                'email_address':  obj.account.email_address,
                'provider':  obj.account.provider,
                # 'status':  'syncing',  # TODO what are values here
                # 'last_sync':  1398790077,  # tuesday 4/29
                # 'scope': ['mail', 'contacts']
            }

        elif isinstance(obj, Message):
            resp = {
                'id':  obj.public_id,
                'object': 'message',
                'ns': obj.namespace.public_id,
                'subject': obj.subject,
                'from': obj.from_addr,
                'to': obj.to_addr,
                'cc': obj.cc_addr,
                'bcc': obj.bcc_addr,
                'date': obj.received_date,
                'thread': obj.thread.public_id,
                'size': obj.size,
                'files': [],  # TODO calculate attachments from blocks
                'body': obj.sanitized_body,
                # 'snippet'     : obj.snippet,
                # 'list_info'   : obj.mailing_list_headers
            }
            if obj.is_draft:
                resp['is_draft'] = obj.is_draft
            return resp

        elif isinstance(obj, Thread):
            return {
                'id':  obj.public_id,
                'object':  'thread',
                'ns':  obj.namespace.public_id,
                'subject':  obj.subject,
                'participants':  obj.participants,
                'recent_date':  obj.recentdate,
                'subject_date': obj.subjectdate,
                'messages':  [m.public_id for m in obj.messages]  # for now
            }

        elif isinstance(obj, Contact):
            return {
                'id': obj.public_id,
                'object': 'contact',
                'ns': obj.namespace.public_id,
                'name': obj.name,
                'email_address': obj.email_address
            }

        elif isinstance(obj, Part):  # ie: Attachments
            return {
                'id': obj.public_id,
                'object': 'file',
                'ns': obj.namespace.public_id,
                'content_type': obj.content_type,
                'size': obj.size,
                'filename': obj.filename or obj.content_id,
                'is_inline': obj.content_disposition is not None
                and obj.content_disposition.lower() == 'inline',
                'message': obj.message.public_id
            }

        elif isinstance(obj, Block):  # ie: Files
            # TODO consider adding more info?
            return {
                'id': obj.public_id,
                'object': 'file',
                'ns': obj.namespace.public_id,
                'content_type': obj.content_type,
                'size': obj.size,
            }

        elif isinstance(obj, User):
            return {
                'id': obj.public_id,
                'object': 'user',
                'name': obj.name,
                'namespaces': [a.namespace.public_id for a in obj.accounts]
                # TOD
            }
            raise NotImplementedError

        elif isinstance(obj, Account):
            # Shouldn't ever need to serialize these...
            raise NotImplementedError

        elif isinstance(obj, SharedFolder):
            raise NotImplementedError

        # elif isinstance(obj, Webhook):
        #     pass

        # elif insstance(obj, CalendarEvent):
        #     pass

        # Let the base class default method raise the TypeError
        return JSONEncoder.default(self, obj)

def cereal(obj, pretty=False):
    if pretty:
        return dumps(obj,
                     sort_keys=True,
                     indent=4,
                     separators=(',', ': '),
                     cls=APIEncoder)
    return dumps(obj, cls=APIEncoder)

def jsonify(obj):
    return Response(cereal(obj, pretty=True),
                    mimetype='application/json')
<commit_msg>Format display name/email pairs in JSON as dicts, not arrays.<commit_after>import datetime
import calendar
from json import JSONEncoder, dumps

from flask import Response

from inbox.server.models.tables.base import (
    Message, SharedFolder, User, Account, Part,
    Contact, Thread, Namespace, Block)

def format_address_list(addresses):
    return [{'name': name, 'email': email} for name, email in addresses]

# Flask's jsonify() doesn't handle datetimes or
# json arrays as primary objects
class APIEncoder(JSONEncoder):

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return calendar.timegm(obj.utctimetuple())

        elif isinstance(obj, Namespace):
            return {
                'id':  obj.public_id,
                'object': 'namepace',
                'ns':  obj.public_id,

                # Account specific
                'account':  obj.account.public_id,
                'email_address':  obj.account.email_address,
                'provider':  obj.account.provider,
                # 'status':  'syncing',  # TODO what are values here
                # 'last_sync':  1398790077,  # tuesday 4/29
                # 'scope': ['mail', 'contacts']
            }

        elif isinstance(obj, Message):
            resp = {
                'id':  obj.public_id,
                'object': 'message',
                'ns': obj.namespace.public_id,
                'subject': obj.subject,
                'from': format_address_list(obj.from_addr),
                'to': format_address_list(obj.to_addr),
                'cc': format_address_list(obj.cc_addr),
                'bcc': format_address_list(obj.bcc_addr),
                'date': obj.received_date,
                'thread': obj.thread.public_id,
                'size': obj.size,
                'files': [],  # TODO calculate attachments from blocks
                'body': obj.sanitized_body,
                # 'snippet'     : obj.snippet,
                # 'list_info'   : obj.mailing_list_headers
            }
            if obj.is_draft:
                resp['is_draft'] = obj.is_draft
            return resp

        elif isinstance(obj, Thread):
            return {
                'id':  obj.public_id,
                'object':  'thread',
                'ns':  obj.namespace.public_id,
                'subject':  obj.subject,
                'participants':  obj.participants,
                'recent_date':  obj.recentdate,
                'subject_date': obj.subjectdate,
                'messages':  [m.public_id for m in obj.messages]  # for now
            }

        elif isinstance(obj, Contact):
            return {
                'id': obj.public_id,
                'object': 'contact',
                'ns': obj.namespace.public_id,
                'name': obj.name,
                'email_address': obj.email_address
            }

        elif isinstance(obj, Part):  # ie: Attachments
            return {
                'id': obj.public_id,
                'object': 'file',
                'ns': obj.namespace.public_id,
                'content_type': obj.content_type,
                'size': obj.size,
                'filename': obj.filename or obj.content_id,
                'is_inline': obj.content_disposition is not None
                and obj.content_disposition.lower() == 'inline',
                'message': obj.message.public_id
            }

        elif isinstance(obj, Block):  # ie: Files
            # TODO consider adding more info?
            return {
                'id': obj.public_id,
                'object': 'file',
                'ns': obj.namespace.public_id,
                'content_type': obj.content_type,
                'size': obj.size,
            }

        elif isinstance(obj, User):
            return {
                'id': obj.public_id,
                'object': 'user',
                'name': obj.name,
                'namespaces': [a.namespace.public_id for a in obj.accounts]
                # TOD
            }
            raise NotImplementedError

        elif isinstance(obj, Account):
            # Shouldn't ever need to serialize these...
            raise NotImplementedError

        elif isinstance(obj, SharedFolder):
            raise NotImplementedError

        # elif isinstance(obj, Webhook):
        #     pass

        # elif insstance(obj, CalendarEvent):
        #     pass

        # Let the base class default method raise the TypeError
        return JSONEncoder.default(self, obj)

def cereal(obj, pretty=False):
    if pretty:
        return dumps(obj,
                     sort_keys=True,
                     indent=4,
                     separators=(',', ': '),
                     cls=APIEncoder)
    return dumps(obj, cls=APIEncoder)

def jsonify(obj):
    return Response(cereal(obj, pretty=True),
                    mimetype='application/json')
<|endoftext|>


<commit_before>from django.contrib import admin

from .admin_actions import mark_paid, mark_received, mark_paid_and_received
from .models import Color, Order, Settings, Size, Type

@admin.register(Type)
class TypeAdmin(admin.ModelAdmin):
    list_display = ['name', 'price']
    ordering = ['-price', 'name']

@admin.register(Size)
class SizeAdmin(admin.ModelAdmin):
    list_display = ['size']
    readonly_fields = ('size_sortable',)

@admin.register(Color)
class ColorAdmin(admin.ModelAdmin):
    list_display = ['name']
    ordering = ['name']

@admin.register(Order)
class OrderAdmin(admin.ModelAdmin):
    list_display = ['person', 'type', 'size', 'color', 'additional']
    ordering = ['person', 'type', 'size']
    list_display_links = ['person', 'type', 'size', 'color']
    list_filter = ['additional', 'color']
    search_fields = ('person__prename', 'person__name')
    readonly_fields = ('created_at', 'updated_at')
    actions = [mark_paid, mark_received, mark_paid_and_received]

@admin.register(Settings)
class SettingsAdmin(admin.ModelAdmin):
    list_display = ('clothing_ordering_enabled',)
<commit_msg>Added filter to clothing order in admin<commit_after>from django.contrib import admin

from .admin_actions import mark_paid, mark_received, mark_paid_and_received
from .models import Color, Order, Settings, Size, Type
from ophasebase.models import Ophase

@admin.register(Type)
class TypeAdmin(admin.ModelAdmin):
    list_display = ['name', 'price']
    ordering = ['-price', 'name']

@admin.register(Size)
class SizeAdmin(admin.ModelAdmin):
    list_display = ['size']
    readonly_fields = ('size_sortable',)

@admin.register(Color)
class ColorAdmin(admin.ModelAdmin):
    list_display = ['name']
    ordering = ['name']

class OphaseOrderFilter(admin.SimpleListFilter):
        """
        A filter to limit the order overview to a specific ophase
        """
        title = 'Ophase'
        parameter_name = 'ophase'

        def lookups(self, request, model_admin):
            """
            Returns a tuple with all ophases
            """
            return ((o, o) for o in Ophase.objects.all())

        def queryset(self, request, queryset):
            if not self.value():
                return queryset
            return queryset.filter(person__ophase=Ophase.objects.get(name=self.value()))

@admin.register(Order)
class OrderAdmin(admin.ModelAdmin):
    list_display = ['person', 'type', 'size', 'color', 'additional']
    ordering = ['person', 'type', 'size']
    list_display_links = ['person', 'type', 'size', 'color']
    list_filter = ['additional', 'color', OphaseOrderFilter]
    search_fields = ('person__prename', 'person__name')
    readonly_fields = ('created_at', 'updated_at')
    actions = [mark_paid, mark_received, mark_paid_and_received]

@admin.register(Settings)
class SettingsAdmin(admin.ModelAdmin):
    list_display = ('clothing_ordering_enabled',)
<|endoftext|>


<commit_before>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import os

from constants import CLOUDIFY_APP_DIR_KEY

MANAGER_IP_KEY = "MANAGEMENT_IP"
LOCAL_IP_KEY = "AGENT_IP"
MANAGER_REST_PORT_KEY = "MANAGER_REST_PORT"

def get_app_dir():
    """
    Returns the directory celery tasks are stored within.
    """
    return os.environ[CLOUDIFY_APP_DIR_KEY]

def get_local_ip():
    return os.environ[LOCAL_IP_KEY]

def get_manager_ip():
    return os.environ[MANAGER_IP_KEY]

def get_manager_rest_service_port():
    return int(os.environ[MANAGER_REST_PORT_KEY])

def get_cosmo_properties():
    return {
        "management_ip": get_manager_ip(),
        "ip": get_local_ip()
    }
<commit_msg>CLOUDIFY-2370 remove method. not used<commit_after>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import os

MANAGER_IP_KEY = "MANAGEMENT_IP"
LOCAL_IP_KEY = "AGENT_IP"
MANAGER_REST_PORT_KEY = "MANAGER_REST_PORT"

def get_local_ip():
    return os.environ[LOCAL_IP_KEY]

def get_manager_ip():
    return os.environ[MANAGER_IP_KEY]

def get_manager_rest_service_port():
    return int(os.environ[MANAGER_REST_PORT_KEY])

def get_cosmo_properties():
    return {
        "management_ip": get_manager_ip(),
        "ip": get_local_ip()
    }
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
"""
Collection of methods to evaluate the results of disteval functions
"""

import numpy as np

from scipy.stats import norm, ks_2samp
from sklearn.metrics import roc_curve

from ..scripts.classifier_characteristics import ClassifierCharacteristics

def feature_importance_mad(clf, alpha=0.05):
    """This function fetches the feature importance values and runs a
    criteria using the median absolute deviation. If a feature
    importance difference to the median importance is greater than
    a certain threshold and the feature is more important than the
    median feature, the feature is removed. The threshold is:
    1.4826 * cdf_norm**-1(1 - alpha/2) * MAD
    The distribution of the feature importance can be expected, to have
    a relativ flat distribution up from 0 upto a normal distributed
    peak. The flat part is for constant or close to constant features.
    The rest of the features can be expected to be choosen in a random
    fashion. Therefore they build a normal distributed peak
    around ~(1. / (n_features - n_constant_features)). To have a robust
    measure for outliers the meadian absolute diviation (MAD) is used.
    The definition of the MAD is:
    median(|X_i - median(X)|)
    For a mormal distribution the 1 sigma region is included in the
    region between 1.4826 * MAD - median(X) and 1.4826 * MAD + median(X).
    With the parameter alpha the used threshold is tuned in a way, for
    a pure normal distribution alpha / 2  (only features above the
    median are removed) features would be removed.

    Parameters
    ----------
    clf: object or list
        Trained classifier or list of trained classifier.

    alpha : float, optional (default=0.05)
        Parameter tuning the threshold. See function describtion.

    Returns
    -------
    kept: numpy.boolarray, shape=(n_features)
        Whether the feature passes the MAD criteria.

    feature_importance: numpy.array, shape=(n_features)
        Array of the importance values for the features. If a list of
        classifier is passed, it is the mean over all classifier.

    feature_importance_std: None or numpy.array, shape=(n_features)
        If a list of classifier is passed the standard deviation is of
        the feature importance values is returned. Otherwise None is
        returned
    """
    desired_characteristics = ClassifierCharacteristics()
    desired_characteristics.opts['has:feature_importances_'] = True

    if isinstance(clf, list):
        feature_importances = []
        for i, clf_i in enumerate(clf):
            clf_characteristics = ClassifierCharacteristics(clf_i)
            assert clf_characteristics.fulfilling(desired_characteristics), \
                'Classifier sanity check failed!'
            feature_importances.append(clf_i.feature_importances_)
        feature_importances = np.array(feature_importances)
        feature_importance = np.mean(feature_importances, axis=0)
        feature_importance_std = np.std(feature_importances, axis=0, ddof=1)
    else:
        got_single_clf = True
        clf_characteristics = ClassifierCharacteristics(clf)
        assert clf_characteristics.fulfilling(desired_characteristics), \
            'Classifier sanity check failed!'
        feature_importance = clf.feature_importances_
        feature_importance_std = np.NaN

    threshold = norm.ppf(1 - alpha/2) * 1.4826 # see docstring
    median_importance = np.median(feature_importance)
    MAD = np.median(np.absolute(feature_importance - median_importance))
    diff = feature_importance - median_importance
    kept = np.logical_or(np.absolute(diff) < threshold * MAD,
                         feature_importance <= median_importance)
    return kept, feature_importance, feature_importance_std

def feature_importance_mad_majority(clfs, ratio=0.9, alpha=0.10):
    """In this function a list of classifier must be provided. To decide
    if a feature is removed, for each classifier the function
    feature_importance_mad with the provided alpha is evaluated. And if
    a feature is removed in atleast ratio-percent of the classifiers
    the feature is removed. The motivation behind the majority vote is,
    that if a feature is just above the threshold in a single test
    because of statistical fluctuation is should be below the threshold
    for most of the classifications. The alpha can be set less
    conservative because this criteria is more robust against
    statistical fluctuationsc.

    Parameters
    ----------
    clf: list
        List of trained classifier.

    ratio : float, optional (default=0.9)
        Ratio of classifiers in which the feature should be removed.

    alpha : float, optional (default=0.05)
        Parameter tuning the threshold. See feature_importance_mad
        describtion.

    Returns
    -------
    kept: numpy.boolarray, shape=(n_features)
        Whether the feature passes the MAD criteria.

    feature_importance: numpy.array, shape=(n_features)
        Array of the importance values for the features. If a list of
        classifier is passed, it is the mean over all classifier.

    feature_importance_std: numpy.array, shape=(n_features)
        If a list of classifier is passed the standard deviation is of
        the feature importance values is returned. Otherwise None is
        returned
    """
    desired_characteristics = ClassifierCharacteristics()
    desired_characteristics.opts['has:feature_importances_'] = True
    assert isinstance(clfs, list), 'List of classifier has to be provided'
    kept_arr = []
    feature_importances = []
    for i, clf_i in enumerate(clfs):
        kept, feature_importance, _ = feature_importance_mad(clf_i,
                                                             alpha=alpha)
        kept_arr.append(kept)
        feature_importances.append(feature_importance)
    kept_arr = np.array(kept_arr)
    feature_importances = np.array(feature_importances)
    feature_importance = np.mean(feature_importances, axis=0)
    feature_importance_std = np.std(feature_importances, axis=0, ddof=1)
    kept = np.sum(kept_arr, axis=0) >= ratio*kept_arr.shape[0]
    return kept, feature_importance, feature_importance_std

def roc_curve_equivalence_ks_test(y_pred_a,
                                  y_pred_b,
                                  y_true,
                                  y_true_b=None,
                                  alpha=0.05):
    """Function evaluating the equivalence between the ROC curves of
    two classifier. The method is described by Andrew P. Bradley in
    "ROC curve equivalence using the Kolmogorov-Smirnov test"
    DOI: 10.1016/j.patrec.2012.12.021

    Parameters
    ----------
    y_pred_a: numpy.array, shape=(n_samples_a)
        Predictions of classifier a. The predictions are expected to be
        between [0, 1].

    y_pred_b: numpy.array, shape=(n_samples_b)
        Predictions of classifier b. he predictions are expected to be
        between [0, 1]. If y_true_b is not provided, the
        sample must be of the same length as sample a.

    y_true : numpy.array, shape=(n_samples_a)
        True labels for sample_a. If y_true_b is not provided, it is
        also used as the true labels for sample b

    y_true_b : None numpy.array, shape=(n_samples_b), optional
        True labels for sample_b. If None y_true_a is used as labels for
        sample b.

    alpha : float, optional (default=0.05)
        Significance for the Kolmogorov Smirnov test.

    Returns
    -------
    passed: bool
        True if test is accepted. False if the test is rejected. A
        rejection has the error rate alpha.

    op_point_a: numpy.array, shape=(2,2)
        [False positive rate, True positive rate] Rate at the operation
        points of both KS test for sample a.

    op_point_b: numpy.array, shape=(2,2)
        [False positive rate, True positive rate] Rate at the operation
        points of both KS test for sample b.

    fpr_b: numpy.array
        False positive rate for sample b at the thresholds.

    tpr_b: numpy.array
        True positive rate for sample b at the thresholds.

    threshold: numpy.array
        Thresholds to the false/true positive rates.
    """

    bincount_y = np.bincount(y_true)
    num_positive_a = bincount_y[1]
    num_negative_a = bincount_y[0]
    if y_true_b is not None:
        bincount_y = np.bincount(y_true_b)
        num_positive_b = bincount_y[1]
        num_negative_b = bincount_y[0]
    else:
        num_positive_b = num_positive_a
        num_negative_b = num_negative_a

    fpr_a, tpr_a, thresholds_a = roc_curve(y_true,
                                           y_pred_a,
                                           drop_intermediate=True)
    fpr_b, tpr_b, thresholds_b = roc_curve(y_true,
                                           y_pred_b,
                                           drop_intermediate=True)

    thresholds = np.sort(np.unique(np.hstack((thresholds_a, thresholds_b))))
    thresholds = thresholds[::-1]
    fpr_a_full = np.ones_like(thresholds)
    tpr_a_full = np.ones_like(thresholds)
    fpr_b_full = np.ones_like(thresholds)
    tpr_b_full = np.ones_like(thresholds)
    pointer_a = -1
    pointer_b = -1

    for i, t_i in enumerate(thresholds):
        if pointer_a+1 < len(thresholds_a):
            if t_i == thresholds_a[pointer_a+1]:
                pointer_a += 1
        if pointer_b+1 < len(thresholds_b):
            if t_i == thresholds_b[pointer_b+1]:
                pointer_b += 1
        fpr_a_full[i] = fpr_a[pointer_a]
        tpr_a_full[i] = tpr_a[pointer_a]
        fpr_b_full[i] = fpr_b[pointer_b]
        tpr_b_full[i] = tpr_b[pointer_b]
    print(fpr_a_full)
    print(tpr_a_full)

    D_n = np.absolute(fpr_a_full - fpr_b_full)
    D_p = np.absolute(tpr_a_full - tpr_b_full)

    idx_max_n = np.argmax(D_n)
    max_D_n = D_n[idx_max_n]
    idx_max_p = np.argmax(D_p)
    max_D_p = D_p[idx_max_p]

    op_point_n = np.array([[fpr_a_full[idx_max_n], fpr_b_full[idx_max_n]],
                           [tpr_a_full[idx_max_n], tpr_b_full[idx_max_n]]])
    op_point_p = np.array([[fpr_a_full[idx_max_p], fpr_b_full[idx_max_p]],
                           [tpr_a_full[idx_max_p], tpr_b_full[idx_max_p]]])

    critical_value = np.sqrt((2. / alpha) / 2)
    passed_test = lambda n,m,d: np.sqrt(n*m / (n + m)) * d > critical_value

    passed = np.logical_and(
        passed_test(num_positive_a, num_positive_b, max_D_p),
        passed_test(num_negative_a, num_negative_b, max_D_n))

    return passed, op_point_n, op_point_p, \
        fpr_a_full, tpr_a_full, fpr_b_full, tpr_b_full, thresholds
<commit_msg>removed debug print statement<commit_after># -*- coding:utf-8 -*-
"""
Collection of methods to evaluate the results of disteval functions
"""

import numpy as np

from scipy.stats import norm
from sklearn.metrics import roc_curve

from ..scripts.classifier_characteristics import ClassifierCharacteristics

def feature_importance_mad(clf, alpha=0.05):
    """This function fetches the feature importance values and runs a
    criteria using the median absolute deviation. If a feature
    importance difference to the median importance is greater than
    a certain threshold and the feature is more important than the
    median feature, the feature is removed. The threshold is:
    1.4826 * cdf_norm**-1(1 - alpha/2) * MAD
    The distribution of the feature importance can be expected, to have
    a relativ flat distribution up from 0 upto a normal distributed
    peak. The flat part is for constant or close to constant features.
    The rest of the features can be expected to be choosen in a random
    fashion. Therefore they build a normal distributed peak
    around ~(1. / (n_features - n_constant_features)). To have a robust
    measure for outliers the meadian absolute diviation (MAD) is used.
    The definition of the MAD is:
    median(|X_i - median(X)|)
    For a mormal distribution the 1 sigma region is included in the
    region between 1.4826 * MAD - median(X) and 1.4826 * MAD + median(X).
    With the parameter alpha the used threshold is tuned in a way, for
    a pure normal distribution alpha / 2  (only features above the
    median are removed) features would be removed.

    Parameters
    ----------
    clf: object or list
        Trained classifier or list of trained classifier.

    alpha : float, optional (default=0.05)
        Parameter tuning the threshold. See function describtion.

    Returns
    -------
    kept: numpy.boolarray, shape=(n_features)
        Whether the feature passes the MAD criteria.

    feature_importance: numpy.array, shape=(n_features)
        Array of the importance values for the features. If a list of
        classifier is passed, it is the mean over all classifier.

    feature_importance_std: None or numpy.array, shape=(n_features)
        If a list of classifier is passed the standard deviation is of
        the feature importance values is returned. Otherwise None is
        returned
    """
    desired_characteristics = ClassifierCharacteristics()
    desired_characteristics.opts['has:feature_importances_'] = True

    if isinstance(clf, list):
        feature_importances = []
        for i, clf_i in enumerate(clf):
            clf_characteristics = ClassifierCharacteristics(clf_i)
            assert clf_characteristics.fulfilling(desired_characteristics), \
                'Classifier sanity check failed!'
            feature_importances.append(clf_i.feature_importances_)
        feature_importances = np.array(feature_importances)
        feature_importance = np.mean(feature_importances, axis=0)
        feature_importance_std = np.std(feature_importances, axis=0, ddof=1)
    else:
        got_single_clf = True
        clf_characteristics = ClassifierCharacteristics(clf)
        assert clf_characteristics.fulfilling(desired_characteristics), \
            'Classifier sanity check failed!'
        feature_importance = clf.feature_importances_
        feature_importance_std = np.NaN

    threshold = norm.ppf(1 - alpha/2) * 1.4826 # see docstring
    median_importance = np.median(feature_importance)
    MAD = np.median(np.absolute(feature_importance - median_importance))
    diff = feature_importance - median_importance
    kept = np.logical_or(np.absolute(diff) < threshold * MAD,
                         feature_importance <= median_importance)
    return kept, feature_importance, feature_importance_std

def feature_importance_mad_majority(clfs, ratio=0.9, alpha=0.10):
    """In this function a list of classifier must be provided. To decide
    if a feature is removed, for each classifier the function
    feature_importance_mad with the provided alpha is evaluated. And if
    a feature is removed in atleast ratio-percent of the classifiers
    the feature is removed. The motivation behind the majority vote is,
    that if a feature is just above the threshold in a single test
    because of statistical fluctuation is should be below the threshold
    for most of the classifications. The alpha can be set less
    conservative because this criteria is more robust against
    statistical fluctuationsc.

    Parameters
    ----------
    clf: list
        List of trained classifier.

    ratio : float, optional (default=0.9)
        Ratio of classifiers in which the feature should be removed.

    alpha : float, optional (default=0.05)
        Parameter tuning the threshold. See feature_importance_mad
        describtion.

    Returns
    -------
    kept: numpy.boolarray, shape=(n_features)
        Whether the feature passes the MAD criteria.

    feature_importance: numpy.array, shape=(n_features)
        Array of the importance values for the features. If a list of
        classifier is passed, it is the mean over all classifier.

    feature_importance_std: numpy.array, shape=(n_features)
        If a list of classifier is passed the standard deviation is of
        the feature importance values is returned. Otherwise None is
        returned
    """
    desired_characteristics = ClassifierCharacteristics()
    desired_characteristics.opts['has:feature_importances_'] = True
    assert isinstance(clfs, list), 'List of classifier has to be provided'
    kept_arr = []
    feature_importances = []
    for i, clf_i in enumerate(clfs):
        kept, feature_importance, _ = feature_importance_mad(clf_i,
                                                             alpha=alpha)
        kept_arr.append(kept)
        feature_importances.append(feature_importance)
    kept_arr = np.array(kept_arr)
    feature_importances = np.array(feature_importances)
    feature_importance = np.mean(feature_importances, axis=0)
    feature_importance_std = np.std(feature_importances, axis=0, ddof=1)
    kept = np.sum(kept_arr, axis=0) >= ratio * kept_arr.shape[0]
    return kept, feature_importance, feature_importance_std

def roc_curve_equivalence_ks_test(y_pred_a,
                                  y_pred_b,
                                  y_true,
                                  y_true_b=None,
                                  alpha=0.05):
    """Function evaluating the equivalence between the ROC curves of
    two classifier. The method is described by Andrew P. Bradley in
    "ROC curve equivalence using the Kolmogorov-Smirnov test"
    DOI: 10.1016/j.patrec.2012.12.021

    Parameters
    ----------
    y_pred_a: numpy.array, shape=(n_samples_a)
        Predictions of classifier a. The predictions are expected to be
        between [0, 1].

    y_pred_b: numpy.array, shape=(n_samples_b)
        Predictions of classifier b. he predictions are expected to be
        between [0, 1]. If y_true_b is not provided, the
        sample must be of the same length as sample a.

    y_true : numpy.array, shape=(n_samples_a)
        True labels for sample_a. If y_true_b is not provided, it is
        also used as the true labels for sample b

    y_true_b : None numpy.array, shape=(n_samples_b), optional
        True labels for sample_b. If None y_true_a is used as labels for
        sample b.

    alpha : float, optional (default=0.05)
        Significance for the Kolmogorov Smirnov test.

    Returns
    -------
    passed: bool
        True if test is accepted. False if the test is rejected. A
        rejection has the error rate alpha.

    op_point_a: numpy.array, shape=(2,2)
        [False positive rate, True positive rate] Rate at the operation
        points of both KS test for sample a.

    op_point_b: numpy.array, shape=(2,2)
        [False positive rate, True positive rate] Rate at the operation
        points of both KS test for sample b.

    fpr_b: numpy.array
        False positive rate for sample b at the thresholds.

    tpr_b: numpy.array
        True positive rate for sample b at the thresholds.

    threshold: numpy.array
        Thresholds to the false/true positive rates.
    """

    bincount_y = np.bincount(y_true)
    num_positive_a = bincount_y[1]
    num_negative_a = bincount_y[0]
    if y_true_b is not None:
        bincount_y = np.bincount(y_true_b)
        num_positive_b = bincount_y[1]
        num_negative_b = bincount_y[0]
    else:
        num_positive_b = num_positive_a
        num_negative_b = num_negative_a

    fpr_a, tpr_a, thresholds_a = roc_curve(y_true,
                                           y_pred_a,
                                           drop_intermediate=True)
    fpr_b, tpr_b, thresholds_b = roc_curve(y_true,
                                           y_pred_b,
                                           drop_intermediate=True)

    thresholds = np.sort(np.unique(np.hstack((thresholds_a, thresholds_b))))
    thresholds = thresholds[::-1]
    fpr_a_full = np.ones_like(thresholds)
    tpr_a_full = np.ones_like(thresholds)
    fpr_b_full = np.ones_like(thresholds)
    tpr_b_full = np.ones_like(thresholds)
    pointer_a = -1
    pointer_b = -1

    for i, t_i in enumerate(thresholds):
        if pointer_a + 1 < len(thresholds_a):
            if t_i == thresholds_a[pointer_a + 1]:
                pointer_a += 1
        if pointer_b + 1 < len(thresholds_b):
            if t_i == thresholds_b[pointer_b + 1]:
                pointer_b += 1
        fpr_a_full[i] = fpr_a[pointer_a]
        tpr_a_full[i] = tpr_a[pointer_a]
        fpr_b_full[i] = fpr_b[pointer_b]
        tpr_b_full[i] = tpr_b[pointer_b]

    D_n = np.absolute(fpr_a_full - fpr_b_full)
    D_p = np.absolute(tpr_a_full - tpr_b_full)

    idx_max_n = np.argmax(D_n)
    max_D_n = D_n[idx_max_n]
    idx_max_p = np.argmax(D_p)
    max_D_p = D_p[idx_max_p]

    op_point_n = np.array([[fpr_a_full[idx_max_n], fpr_b_full[idx_max_n]],
                           [tpr_a_full[idx_max_n], tpr_b_full[idx_max_n]]])
    op_point_p = np.array([[fpr_a_full[idx_max_p], fpr_b_full[idx_max_p]],
                           [tpr_a_full[idx_max_p], tpr_b_full[idx_max_p]]])

    critical_value = np.sqrt((2. / np.sqrt(alpha)) / 2)
    passed_test = lambda n, m, d: np.sqrt(n * m / (n + m)) * d > critical_value

    passed = np.logical_and(
        passed_test(num_positive_a, num_positive_b, max_D_p),
        passed_test(num_negative_a, num_negative_b, max_D_n))

    return passed, op_point_n, op_point_p, \
        fpr_a_full, tpr_a_full, fpr_b_full, tpr_b_full, thresholds
<|endoftext|>


<commit_before>"""
Some estimators based on estimating local densities using k-nearest neighbors.
"""

from __future__ import division

import numpy as np

from scipy.special import digamma

from six.moves import zip

from dit.utils import flatten

try:
    from sklearn.neighbors import KDTree
    total_correlation_ksg = _total_correlation_ksg_sklearn
except ImportError:
    from scipy.spatial import cKDTree
    total_correlation_ksg = _total_correlation_ksg_scipy

__all__ = ['differential_entropy_knn',
           'total_correlation_ksg',
          ]

def _fuzz(data, noise):
    """
    Add noise to the data.

    Parameters
    ----------
    data : np.ndarray
        Data.
    noise : float
        The standard deviation of the normally-distributed noise to add to data.

    Returns
    -------
    data : np.ndarray
        The fuzzed data.
    """
    data = data.astype(np.float)
    data += np.random.normal(0.0, noise, size=data.shape)
    return data

def differential_entropy_knn(data, rvs=None, k=4, noise=1e-10):
    """
    Compute the *differential* entropy of `data` using a k-nearest neighbors density estimator.

    Parameters
    ----------
    data : np.ndarray
        The data.
    rvs : list
        The columns of `data` to use as the random variable. If None, use all.
    k : int
        The number of nearest neighbors to use.

    Returns
    -------
    h : float
        The estimated entropy.

    Notes
    -----
    The entropy is returned in units of bits.
    """
    if rvs is None:
        rvs = list(range(data.shape[1]))

    data = _fuzz(data, noise)

    d = len(rvs)

    tree = cKDTree(data[:, rvs])

    epsilons = tree.query(data[:, rvs], k + 1, p=np.inf)[0][:, -1]

    h = digamma(len(data)) - digamma(k) + d * (np.log(2) + np.log(epsilons).mean())

    return h / np.log(2)

def _total_correlation_ksg_scipy(data, rvs, crvs=None, k=4, noise=1e-10):
    """
    Compute the total correlation from observations. The total correlation is computed between the columns
    specified in `rvs`, given the columns specified in `crvs`. This utilizes the KSG kNN density estimator,
    and works on discrete, continuous, and mixed data.

    Parameters
    ----------
    data : np.array
        A set of observations of a distribution.
    rvs : iterable of iterables
        The columns for which the total correlation is to be computed.
    crvs : iterable
        The columns upon which the total correlation should be conditioned.
    k : int
        The number of nearest neighbors to use in estimating the local kernel density.
    noise : float
        The standard deviation of the normally-distributed noise to add to the data.

    Returns
    -------
    tc : float
        The total correlation of `rvs` given `crvs`.

    Notes
    -----
    The total correlation is computed in bits, not nats as most KSG estimators do.
    """
    # KSG suggest adding noise (to break symmetries?)
    data = _fuzz(data, noise)

    if crvs is None:
        crvs = []

    digamma_N = digamma(len(data))
    log_2 = np.log(2)

    all_rvs = list(flatten(rvs)) + crvs
    rvs = [rv + crvs for rv in rvs]

    d_rvs = [len(data[0, rv]) for rv in rvs]

    tree = cKDTree(data[:, all_rvs])
    tree_rvs = [cKDTree(data[:, rv]) for rv in rvs]

    epsilons = tree.query(data[:, all_rvs], k + 1, p=np.inf)[0][:, -1]  # k+1 because of self

    n_rvs = [
        np.array([len(t.query_ball_point(point, epsilon, p=np.inf)) for point, epsilon in zip(data[:, rv], epsilons)])
        for rv, t in zip(rvs, tree_rvs)]

    log_epsilons = np.log(epsilons)

    h_rvs = [-digamma(n_rv).mean() for n_rv, d in zip(n_rvs, d_rvs)]

    h_all = -digamma(k)

    if crvs:
        tree_crvs = cKDTree(data[:, crvs])
        n_crvs = np.array([len(tree_crvs.query_ball_point(point, epsilon, p=np.inf)) for point, epsilon in
                           zip(data[:, crvs], epsilons)])
        h_crvs = -digamma(n_crvs).mean()
    else:
        h_rvs = [h_rv + digamma_N + d * (log_2 - log_epsilons).mean() for h_rv, d in zip(h_rvs, d_rvs)]
        h_all += digamma_N + sum(d_rvs) * (log_2 - log_epsilons).mean()
        h_crvs = 0

    tc = sum([h_rv - h_crvs for h_rv in h_rvs]) - (h_all - h_crvs)

    return tc / log_2

def _total_correlation_ksg_sklearn(data, rvs, crvs=None, k=4, noise=1e-10):
    """
    Compute the total correlation from observations. The total correlation is computed between the columns
    specified in `rvs`, given the columns specified in `crvs`. This utilizes the KSG kNN density estimator,
    and works on discrete, continuous, and mixed data.

    Parameters
    ----------
    data : np.array
        Real valued time series data.
    rvs : iterable of iterables
        The columns for which the total correlation is to be computed.
    crvs : iterable
        The columns upon which the total correlation should be conditioned.
    k : int
        The number of nearest neighbors to use in estimating the local kernel density.
    noise : float
        The standard deviation of the normally-distributed noise to add to the data.

    Returns
    -------
    tc : float
        The total correlation of `rvs` given `crvs`.

    Notes
    -----
    The total correlation is computed in bits, not nats as most KSG estimators do.

    This implementation uses scikit-learn.
    """
    # KSG suggest adding noise (to break symmetries?)
    data = _fuzz(data, noise)

    if crvs is None:
        crvs = []

    digamma_N = digamma(len(data))
    log_2 = np.log(2)

    all_rvs = list(flatten(rvs)) + crvs
    rvs = [rv + crvs for rv in rvs]

    d_rvs = [len(data[0, rv]) for rv in rvs]

    tree = KDTree(data[:, all_rvs], metric="chebyshev")
    tree_rvs = [KDTree(data[:, rv], metric="chebyshev") for rv in rvs]

    epsilons = tree.query(data[:, all_rvs], k + 1)[0][:, -1]  # k+1 because of self

    n_rvs = [t.query_radius(data[:, rv], epsilons, count_only=True) for rv, t in zip(rvs, tree_rvs)]

    log_epsilons = np.log(epsilons)

    h_rvs = [-digamma(n_rv).mean() for n_rv, d in zip(n_rvs, d_rvs)]

    h_all = -digamma(k)

    if crvs:
        tree_crvs = KDTree(data[:, crvs], metric="chebyshev")
        n_crvs = tree_crvs.query_radius(data[:, crvs], epsilons, count_only=True)
        h_crvs = -digamma(n_crvs).mean()
    else:
        h_rvs = [h_rv + digamma_N + d * (log_2 - log_epsilons).mean() for h_rv, d in zip(h_rvs, d_rvs)]
        h_all += digamma_N + sum(d_rvs) * (log_2 - log_epsilons).mean()
        h_crvs = 0

    tc = sum([h_rv - h_crvs for h_rv in h_rvs]) - (h_all - h_crvs)

    return tc / log_2
<commit_msg>fix ksg method selection<commit_after>"""
Some estimators based on estimating local densities using k-nearest neighbors.
"""

from __future__ import division

import numpy as np

from scipy.special import digamma

from six.moves import zip

from dit.utils import flatten

__all__ = ['differential_entropy_knn',
           'total_correlation_ksg',
          ]

def _fuzz(data, noise):
    """
    Add noise to the data.

    Parameters
    ----------
    data : np.ndarray
        Data.
    noise : float
        The standard deviation of the normally-distributed noise to add to data.

    Returns
    -------
    data : np.ndarray
        The fuzzed data.
    """
    data = data.astype(np.float)
    data += np.random.normal(0.0, noise, size=data.shape)
    return data

def differential_entropy_knn(data, rvs=None, k=4, noise=1e-10):
    """
    Compute the *differential* entropy of `data` using a k-nearest neighbors density estimator.

    Parameters
    ----------
    data : np.ndarray
        The data.
    rvs : list
        The columns of `data` to use as the random variable. If None, use all.
    k : int
        The number of nearest neighbors to use.

    Returns
    -------
    h : float
        The estimated entropy.

    Notes
    -----
    The entropy is returned in units of bits.
    """
    if rvs is None:
        rvs = list(range(data.shape[1]))

    data = _fuzz(data, noise)

    d = len(rvs)

    tree = cKDTree(data[:, rvs])

    epsilons = tree.query(data[:, rvs], k + 1, p=np.inf)[0][:, -1]

    h = digamma(len(data)) - digamma(k) + d * (np.log(2) + np.log(epsilons).mean())

    return h / np.log(2)

def _total_correlation_ksg_scipy(data, rvs, crvs=None, k=4, noise=1e-10):
    """
    Compute the total correlation from observations. The total correlation is computed between the columns
    specified in `rvs`, given the columns specified in `crvs`. This utilizes the KSG kNN density estimator,
    and works on discrete, continuous, and mixed data.

    Parameters
    ----------
    data : np.array
        A set of observations of a distribution.
    rvs : iterable of iterables
        The columns for which the total correlation is to be computed.
    crvs : iterable
        The columns upon which the total correlation should be conditioned.
    k : int
        The number of nearest neighbors to use in estimating the local kernel density.
    noise : float
        The standard deviation of the normally-distributed noise to add to the data.

    Returns
    -------
    tc : float
        The total correlation of `rvs` given `crvs`.

    Notes
    -----
    The total correlation is computed in bits, not nats as most KSG estimators do.
    """
    # KSG suggest adding noise (to break symmetries?)
    data = _fuzz(data, noise)

    if crvs is None:
        crvs = []

    digamma_N = digamma(len(data))
    log_2 = np.log(2)

    all_rvs = list(flatten(rvs)) + crvs
    rvs = [rv + crvs for rv in rvs]

    d_rvs = [len(data[0, rv]) for rv in rvs]

    tree = cKDTree(data[:, all_rvs])
    tree_rvs = [cKDTree(data[:, rv]) for rv in rvs]

    epsilons = tree.query(data[:, all_rvs], k + 1, p=np.inf)[0][:, -1]  # k+1 because of self

    n_rvs = [
        np.array([len(t.query_ball_point(point, epsilon, p=np.inf)) for point, epsilon in zip(data[:, rv], epsilons)])
        for rv, t in zip(rvs, tree_rvs)]

    log_epsilons = np.log(epsilons)

    h_rvs = [-digamma(n_rv).mean() for n_rv, d in zip(n_rvs, d_rvs)]

    h_all = -digamma(k)

    if crvs:
        tree_crvs = cKDTree(data[:, crvs])
        n_crvs = np.array([len(tree_crvs.query_ball_point(point, epsilon, p=np.inf)) for point, epsilon in
                           zip(data[:, crvs], epsilons)])
        h_crvs = -digamma(n_crvs).mean()
    else:
        h_rvs = [h_rv + digamma_N + d * (log_2 - log_epsilons).mean() for h_rv, d in zip(h_rvs, d_rvs)]
        h_all += digamma_N + sum(d_rvs) * (log_2 - log_epsilons).mean()
        h_crvs = 0

    tc = sum([h_rv - h_crvs for h_rv in h_rvs]) - (h_all - h_crvs)

    return tc / log_2

def _total_correlation_ksg_sklearn(data, rvs, crvs=None, k=4, noise=1e-10):
    """
    Compute the total correlation from observations. The total correlation is computed between the columns
    specified in `rvs`, given the columns specified in `crvs`. This utilizes the KSG kNN density estimator,
    and works on discrete, continuous, and mixed data.

    Parameters
    ----------
    data : np.array
        Real valued time series data.
    rvs : iterable of iterables
        The columns for which the total correlation is to be computed.
    crvs : iterable
        The columns upon which the total correlation should be conditioned.
    k : int
        The number of nearest neighbors to use in estimating the local kernel density.
    noise : float
        The standard deviation of the normally-distributed noise to add to the data.

    Returns
    -------
    tc : float
        The total correlation of `rvs` given `crvs`.

    Notes
    -----
    The total correlation is computed in bits, not nats as most KSG estimators do.

    This implementation uses scikit-learn.
    """
    # KSG suggest adding noise (to break symmetries?)
    data = _fuzz(data, noise)

    if crvs is None:
        crvs = []

    digamma_N = digamma(len(data))
    log_2 = np.log(2)

    all_rvs = list(flatten(rvs)) + crvs
    rvs = [rv + crvs for rv in rvs]

    d_rvs = [len(data[0, rv]) for rv in rvs]

    tree = KDTree(data[:, all_rvs], metric="chebyshev")
    tree_rvs = [KDTree(data[:, rv], metric="chebyshev") for rv in rvs]

    epsilons = tree.query(data[:, all_rvs], k + 1)[0][:, -1]  # k+1 because of self

    n_rvs = [t.query_radius(data[:, rv], epsilons, count_only=True) for rv, t in zip(rvs, tree_rvs)]

    log_epsilons = np.log(epsilons)

    h_rvs = [-digamma(n_rv).mean() for n_rv, d in zip(n_rvs, d_rvs)]

    h_all = -digamma(k)

    if crvs:
        tree_crvs = KDTree(data[:, crvs], metric="chebyshev")
        n_crvs = tree_crvs.query_radius(data[:, crvs], epsilons, count_only=True)
        h_crvs = -digamma(n_crvs).mean()
    else:
        h_rvs = [h_rv + digamma_N + d * (log_2 - log_epsilons).mean() for h_rv, d in zip(h_rvs, d_rvs)]
        h_all += digamma_N + sum(d_rvs) * (log_2 - log_epsilons).mean()
        h_crvs = 0

    tc = sum([h_rv - h_crvs for h_rv in h_rvs]) - (h_all - h_crvs)

    return tc / log_2

try:
    from sklearn.neighbors import KDTree
    total_correlation_ksg = _total_correlation_ksg_sklearn
except ImportError:
    from scipy.spatial import cKDTree
    total_correlation_ksg = _total_correlation_ksg_scipy
<|endoftext|>


<commit_before>"""
Built-in, globally-available admin actions.
"""

from django import template
from django.core.exceptions import PermissionDenied
from django.contrib.admin import helpers
from django.contrib.admin.util import get_deleted_objects, model_ngettext
from django.shortcuts import render_to_response
from django.utils.encoding import force_unicode
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy, ugettext as _

def delete_selected(modeladmin, request, queryset):
    """
    Default action which deletes the selected objects.

    This action first displays a confirmation page whichs shows all the
    deleteable objects, or, if the user has no permission one of the related
    childs (foreignkeys), a "permission denied" message.

    Next, it delets all selected objects and redirects back to the change list.
    """
    opts = modeladmin.model._meta
    app_label = opts.app_label

    # Check that the user has delete permission for the actual model
    if not modeladmin.has_delete_permission(request):
        raise PermissionDenied

    # Populate deletable_objects, a data structure of all related objects that
    # will also be deleted.

    # deletable_objects must be a list if we want to use '|unordered_list' in the template
    deletable_objects = []
    perms_needed = set()
    i = 0
    for obj in queryset:
        deletable_objects.append([mark_safe(u'%s: %s' % (escape(force_unicode(capfirst(opts.verbose_name))), obj.pk, escape(obj))), []])
        get_deleted_objects(deletable_objects[i], perms_needed, request.user, obj, opts, 1, modeladmin.admin_site, levels_to_root=2)
        i=i+1

    # The user has already confirmed the deletion.
    # Do the deletion and return a None to display the change list view again.
    if request.POST.get('post'):
        if perms_needed:
            raise PermissionDenied
        n = queryset.count()
        if n:
            for obj in queryset:
                obj_display = force_unicode(obj)
                modeladmin.log_deletion(request, obj, obj_display)
            queryset.delete()
            modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % {
                "count": n, "items": model_ngettext(modeladmin.opts, n)
            })
        # Return None to display the change list page again.
        return None

    context = {
        "title": _("Are you sure?"),
        "object_name": force_unicode(opts.verbose_name),
        "deletable_objects": deletable_objects,
        'queryset': queryset,
        "perms_lacking": perms_needed,
        "opts": opts,
        "root_path": modeladmin.admin_site.root_path,
        "app_label": app_label,
        'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
    }

    # Display the confirmation page
    return render_to_response(modeladmin.delete_confirmation_template or [
        "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.object_name.lower()),
        "admin/%s/delete_selected_confirmation.html" % app_label,
        "admin/delete_selected_confirmation.html"
    ], context, context_instance=template.RequestContext(request))

delete_selected.short_description = ugettext_lazy("Delete selected %(verbose_name_plural)s")
<commit_msg>Fixed #10997: fixed a Python 2.4-ism in admin actions.<commit_after>"""
Built-in, globally-available admin actions.
"""

from django import template
from django.core.exceptions import PermissionDenied
from django.contrib.admin import helpers
from django.contrib.admin.util import get_deleted_objects, model_ngettext
from django.shortcuts import render_to_response
from django.utils.encoding import force_unicode
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy, ugettext as _
try:
    set
except NameError:
    from sets import Set as set     # Python 2.3 fallback

def delete_selected(modeladmin, request, queryset):
    """
    Default action which deletes the selected objects.

    This action first displays a confirmation page whichs shows all the
    deleteable objects, or, if the user has no permission one of the related
    childs (foreignkeys), a "permission denied" message.

    Next, it delets all selected objects and redirects back to the change list.
    """
    opts = modeladmin.model._meta
    app_label = opts.app_label

    # Check that the user has delete permission for the actual model
    if not modeladmin.has_delete_permission(request):
        raise PermissionDenied

    # Populate deletable_objects, a data structure of all related objects that
    # will also be deleted.

    # deletable_objects must be a list if we want to use '|unordered_list' in the template
    deletable_objects = []
    perms_needed = set()
    i = 0
    for obj in queryset:
        deletable_objects.append([mark_safe(u'%s: %s' % (escape(force_unicode(capfirst(opts.verbose_name))), obj.pk, escape(obj))), []])
        get_deleted_objects(deletable_objects[i], perms_needed, request.user, obj, opts, 1, modeladmin.admin_site, levels_to_root=2)
        i=i+1

    # The user has already confirmed the deletion.
    # Do the deletion and return a None to display the change list view again.
    if request.POST.get('post'):
        if perms_needed:
            raise PermissionDenied
        n = queryset.count()
        if n:
            for obj in queryset:
                obj_display = force_unicode(obj)
                modeladmin.log_deletion(request, obj, obj_display)
            queryset.delete()
            modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % {
                "count": n, "items": model_ngettext(modeladmin.opts, n)
            })
        # Return None to display the change list page again.
        return None

    context = {
        "title": _("Are you sure?"),
        "object_name": force_unicode(opts.verbose_name),
        "deletable_objects": deletable_objects,
        'queryset': queryset,
        "perms_lacking": perms_needed,
        "opts": opts,
        "root_path": modeladmin.admin_site.root_path,
        "app_label": app_label,
        'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
    }

    # Display the confirmation page
    return render_to_response(modeladmin.delete_confirmation_template or [
        "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.object_name.lower()),
        "admin/%s/delete_selected_confirmation.html" % app_label,
        "admin/delete_selected_confirmation.html"
    ], context, context_instance=template.RequestContext(request))

delete_selected.short_description = ugettext_lazy("Delete selected %(verbose_name_plural)s")
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import logging
import random

import CSVReader
from DecisionTree import DecisionTreeClassifier

"""
TODO:   - add param size of feature selection in tree
        - implement a thread pool to fit multiple trees at the same time
        - unittests
"""

class RandomForestClassifier(object):

    """
    :param  nb_trees:   Number of decision trees to use
    :param  nb_samples: Number of samples to give to each tree
    :param  max_depth:  Maximum depth of the trees
    """
    def __init__(self, nb_trees, nb_samples, max_depth=-1):
        self.trees = []
        self.nb_trees = nb_trees
        self.nb_samples = nb_samples
        self.max_depth = max_depth

    """
    Trains self.nb_trees number of decision trees.
    :param  data:   A list of lists with the last element of each list being
                    the value to predict
    """
    def fit(self, data):
        for i in range(self.nb_trees):
            logging.info('Training tree {}'.format(i + 1))

            random_features = random.sample(data, self.nb_samples)
            tree = DecisionTreeClassifier(max_depth=self.max_depth)
            tree.fit(random_features)

            self.trees.append(tree)

    """
    Returns a prediction for the given feature. The result is the value that
    gets the most votes.
    :param  feature:    The features used to predict
    """
    def predict(self, feature):
        predictions = []

        for tree in self.trees:
            predictions.append(tree.predict(feature))

        return max(set(predictions), key=predictions.count)

def test_rf():
    from sklearn.model_selection import train_test_split

    data = CSVReader.read_csv("../scala/data/income.csv")
    train, test = train_test_split(data, test_size=0.3)

    rf = RandomForestClassifier(nb_trees=60, nb_samples=3000)
    rf.fit(train)

    errors = 0
    features = [ft[:-1] for ft in test]
    values = [ft[-1] for ft in test]

    for feature, value in zip(features, values):
        prediction = rf.predict(feature)
        if prediction != value:
            errors += 1

    logging.info("Error rate: {}".format(errors / len(features) * 100))

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    test_rf()
<commit_msg>preocess pool to train the trees<commit_after>#!/usr/bin/env python3

import logging
import random
from concurrent.futures import ProcessPoolExecutor

import CSVReader
from DecisionTree import DecisionTreeClassifier

"""
TODO:   - add param size of feature selection in tree
        - unittests
"""

class RandomForestClassifier(object):

    """
    :param  nb_trees:       Number of decision trees to use
    :param  nb_samples:     Number of samples to give to each tree
    :param  max_depth:      Maximum depth of the trees
    :param  max_workers:    Maximum number of processes to use for training
    """
    def __init__(self, nb_trees, nb_samples, max_depth=-1, max_workers=1):
        self.trees = []
        self.nb_trees = nb_trees
        self.nb_samples = nb_samples
        self.max_depth = max_depth
        self.max_workers = max_workers

    """
    Trains self.nb_trees number of decision trees.
    :param  data:   A list of lists with the last element of each list being
                    the value to predict
    """
    def fit(self, data):
        with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
            rand_fts = map(lambda x: [x, random.sample(data, self.nb_samples)],
                           range(self.nb_trees))
            self.trees = list(executor.map(self.train_tree, rand_fts))

    """
    Trains a single tree and returns it.
    :param  data:   A List containing the index of the tree being trained
                    and the data to train it
    """
    def train_tree(self, data):
        logging.info('Training tree {}'.format(data[0] + 1))
        tree = DecisionTreeClassifier(max_depth=self.max_depth)
        tree.fit(data[1])
        return tree

    """
    Returns a prediction for the given feature. The result is the value that
    gets the most votes.
    :param  feature:    The features used to predict
    """
    def predict(self, feature):
        predictions = []

        for tree in self.trees:
            predictions.append(tree.predict(feature))

        return max(set(predictions), key=predictions.count)

def test_rf():
    from sklearn.model_selection import train_test_split

    data = CSVReader.read_csv("../scala/data/income.csv")
    train, test = train_test_split(data, test_size=0.3)

    rf = RandomForestClassifier(nb_trees=60, nb_samples=3000, max_workers=4)
    rf.fit(train)

    errors = 0
    features = [ft[:-1] for ft in test]
    values = [ft[-1] for ft in test]

    for feature, value in zip(features, values):
        prediction = rf.predict(feature)
        if prediction != value:
            errors += 1

    logging.info("Error rate: {}".format(errors / len(features) * 100))

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    test_rf()
<|endoftext|>