<commit_before>#!/usr/bin/env python
#
# Copyright 2011 Map Of Life
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from layers.lib.base import BaseController
from layers.lib.taskqueue import worker_q
from layers.lib.mol.service import GenerateTiles
from pylons import response, request, app_globals
import logging
import os
import simplejson
from layers.lib.mol.service import Layer
from layers.cando import tiling
import math

log = logging.getLogger(__name__)

class ApiController(BaseController):

    def testid(self):
        id = request.params.get('id', None)
        url = request.params.get('url', None)
        return simplejson.dumps({'species_id':id, 'valid':Layer.isidvalid(id, url)})

    def tiles(self, species_id, zoom, x, y):
        '''This action returns PNG data with the response header Content-Type
        set to image/png with a 200 status code. It handles URLs of the form:

        http://mol.colorado.edu/tiles/species_id/zoom/x/y.png

        If a tile isn't found a 404 status is returned.
        '''
        png = os.path.join(app_globals.TILE_DIR, species_id, zoom, x, y + '.png')
        if os.path.exists(png):
            logging.info('Returning tile : ' + png)
            response.headers['Content-Type'] = 'image/png'
            response.status = 200
            return open(png, 'rb').read()
        logging.info('Tile not found : ' + png)
        response.status = 404

    def ecoregion(self, method, name):
        if method=='tile':
            overwrite = False
            try:
                if str(request.GET['overwrite']).strip().lower() == 'true':
                    overwrite = True
            except:
                pass
            newmapfile = False
            try:
                if str(request.GET['newmapfile']).strip().lower() == 'true':
                    newmapfile = True
            except:
                pass
            logging.error(overwrite)
            x = request.GET['x']
            y = request.GET['y']
            z = request.GET['zoom']
            name = name.strip('.png')

            png = os.path.join(app_globals.ECOTILE_DIR, name, z, x, y + '.png')
            nullPng = os.path.join(app_globals.ECOTILE_DIR, name, z, x, y + '.null')

            if overwrite or (not os.path.exists(png) and not os.path.exists(nullPng)):
                try:
                    region_ids = request.GET['region_ids'].split(',')
                except:
                    region_ids = [name]
                tmp_xml = """<?xml version="1.0" encoding="utf-8"?>
                                <!DOCTYPE Map>
                                <Map bgcolor="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +over +no_defs">
                                  <Style name="style">
                                    <Rule>
                                      <PolygonSymbolizer>
                                        <CssParameter name="fill">#000000</CssParameter>
                                      </PolygonSymbolizer>
                                      <LineSymbolizer>
                                        <CssParameter name="stroke">#000000</CssParameter>
                                        <CssParameter name="stroke-width">0.0</CssParameter>
                                      </LineSymbolizer>
                                    </Rule>
                                  </Style>"""
                mapfile = str(app_globals.ECOSHP_DIR +'/'+ name + '.mapfile.xml')

                logging.info('Creating mapfile: %s' % (mapfile))

                pixels = 256

                res = (2 * math.pi * 6378137 / pixels) / (2**int(z))
                sh = 2 * math.pi * 6378137 / 2.0

                gy = (2**float(z) - 1) - int(y)
                minx, miny = ((float(x)*pixels) * res - sh),      (((float(gy))*pixels) * res - sh)
                maxx, maxy = (((float(x)+1)*pixels) * res - sh),  (((float(gy)+1)*pixels) * res - sh)

                minx, maxx = (minx / sh) * 180.0, (maxx / sh) * 180.0
                miny, maxy = (miny / sh) * 180.0, (maxy / sh) * 180.0
                miny = 180 / math.pi * (2 * math.atan( math.exp( miny * math.pi / 180.0)) - math.pi / 2.0)
                maxy = 180 / math.pi * (2 * math.atan( math.exp( maxy * math.pi / 180.0)) - math.pi / 2.0)

                bbox = (minx, miny, maxx, maxy)

                logging.info('Tiling %s with bbox: %s' % (name,str(bbox)))

                ct=0
                for id in region_ids:
                    shp = os.path.join(app_globals.ECOSHP_DIR, id + '.shp')
                    if os.path.exists(shp):
                        ct+=1
                        tmp_xml += """
                                <Layer name="layer_name" srs="+proj=latlong +datum=WGS84">
                                <StyleName>style</StyleName>
                                <Datasource>
                                  <Parameter name="type">shape</Parameter>
                                  <Parameter name="file">layer_name</Parameter>
                                </Datasource>
                              </Layer>""".replace("layer_name",id)

                tmp_xml += "</Map>"
                if ct>0:
                    tile_dir =  str(app_globals.ECOTILE_DIR.rstrip('/') + "/" + name +"/")
                    if newmapfile or not os.path.exists(mapfile):
                        open(mapfile, "w+").write(tmp_xml)
                    if not os.path.isdir(tile_dir):
                        os.mkdir(tile_dir)
                    GenerateTiles.render_tiles(bbox,
                                               mapfile,
                                               tile_dir,
                                               int(z),
                                               int(z),
                                               name,
                                               num_threads=app_globals.TILE_QUEUE_THREADS,
                                               overwrite=overwrite)
            if os.path.exists(png):
                logging.error('Returning tile : ' + png)
                response.headers['Content-Type'] = 'image/png'
                response.status = 200
                return open(png, 'rb').read()

            logging.info('Tile not found : ' + png)
            response.status = 404

        elif str(method).lower() == 'tilearea':
            """ Takes one to many record_ids split by commas and a unique
                name for the tileset you are creating.

                e.g., If you are creating a tileset for a single ecoregion
                with code NT105, you would probably want to create a new
                tileset with,
                    name = 'NT105',
                    record_ids = 'NT105'
                If you are creating a new tileset for hyp. spec 'Andreus
                andreus' you could create a tileset with,
                    name = 'animalia-mammalia-Andreus_andreus'
                    record_ids = 'NT103','NT445','NT6742'
            """

            try:
                region_ids = request.GET['region_ids'].split(',')
            except:
                region_ids = [name]
            lowx = request.GET['lowx']
            lowy = request.GET['lowy']
            highx = request.GET['highx']
            highy = request.GET['highy']
            zoom = int(request.GET['zoom'])

            request_ip = request.environ['REMOTE_ADDR']
            if request_ip != "127.0.0.1":
                tiling.tasks.EcoregionProcessingThread.apply_async(args=[name, zoom, lowx, lowy, highx, highy, region_ids])
                logging.info('Sending job to Celery')
                response.status = 200
                return
            else:
                tile_dir =  str(app_globals.ECOTILE_DIR.rstrip('/') + "/" + name +"/")

                tmp_xml = """<?xml version="1.0" encoding="utf-8"?>
                                <!DOCTYPE Map>
                                <Map bgcolor="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +over +no_defs">
                                  <Style name="style">
                                    <Rule>
                                      <PolygonSymbolizer>
                                        <CssParameter name="fill">#000000</CssParameter>
                                      </PolygonSymbolizer>
                                      <LineSymbolizer>
                                        <CssParameter name="stroke">#000000</CssParameter>
                                        <CssParameter name="stroke-width">0.0</CssParameter>
                                      </LineSymbolizer>
                                    </Rule>
                                  </Style>"""
                mapfile = str(app_globals.ECOSHP_DIR +'/'+ name + '.mapfile.xml')

                logging.info('Creating mapfile: %s' % (mapfile))

                bbox = (float(lowx), float(lowy), float(highx), float(highy))
                logging.info('Tiling %s with bbox: %s' % (name,str(bbox)))

                ct=0
                for id in region_ids:
                    shp = os.path.join(app_globals.ECOSHP_DIR, id + '.shp')
                    if os.path.exists(shp):
                        ct+=1
                        tmp_xml += """
                                <Layer name="layer_name" srs="+proj=latlong +datum=WGS84">
                                <StyleName>style</StyleName>
                                <Datasource>
                                  <Parameter name="type">shape</Parameter>
                                  <Parameter name="file">layer_name</Parameter>
                                </Datasource>
                              </Layer>""".replace("layer_name",id)

                tmp_xml += "</Map>"
                if ct>0:
                    open(mapfile, "w+").write(tmp_xml)
                    if not os.path.isdir(tile_dir):
                        os.mkdir(tile_dir)
                    GenerateTiles.render_tiles(bbox,
                                               mapfile,
                                               tile_dir,
                                               zoom,
                                               zoom,
                                               name,
                                               num_threads=app_globals.TILE_QUEUE_THREADS,
                                               overwrite=False)
                    response.status = 200
                    return

                logging.info('Regions not found : ' + shp)
                response.status = 404
<commit_msg>cando tiling queue<commit_after>#!/usr/bin/env python
#
# Copyright 2011 Map Of Life
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from layers.lib.base import BaseController
from layers.lib.taskqueue import worker_q
from layers.lib.mol.service import GenerateTiles
from pylons import response, request, app_globals
import logging
import os
import simplejson
from layers.lib.mol.service import Layer
from layers.cando.tiling import tasks
import math

log = logging.getLogger(__name__)

class ApiController(BaseController):

    def testid(self):
        id = request.params.get('id', None)
        url = request.params.get('url', None)
        return simplejson.dumps({'species_id':id, 'valid':Layer.isidvalid(id, url)})

    def tiles(self, species_id, zoom, x, y):
        '''This action returns PNG data with the response header Content-Type
        set to image/png with a 200 status code. It handles URLs of the form:

        http://mol.colorado.edu/tiles/species_id/zoom/x/y.png

        If a tile isn't found a 404 status is returned.
        '''
        png = os.path.join(app_globals.TILE_DIR, species_id, zoom, x, y + '.png')
        if os.path.exists(png):
            logging.info('Returning tile : ' + png)
            response.headers['Content-Type'] = 'image/png'
            response.status = 200
            return open(png, 'rb').read()
        logging.info('Tile not found : ' + png)
        response.status = 404

    def ecoregion(self, method, name):
        if method=='tile':
            overwrite = False
            try:
                if str(request.GET['overwrite']).strip().lower() == 'true':
                    overwrite = True
            except:
                pass
            newmapfile = False
            try:
                if str(request.GET['newmapfile']).strip().lower() == 'true':
                    newmapfile = True
            except:
                pass
            logging.error(overwrite)
            x = request.GET['x']
            y = request.GET['y']
            z = request.GET['zoom']
            name = name.strip('.png')

            png = os.path.join(app_globals.ECOTILE_DIR, name, z, x, y + '.png')
            nullPng = os.path.join(app_globals.ECOTILE_DIR, name, z, x, y + '.null')

            if overwrite or (not os.path.exists(png) and not os.path.exists(nullPng)):
                try:
                    region_ids = request.GET['region_ids'].split(',')
                except:
                    region_ids = [name]
                tmp_xml = """<?xml version="1.0" encoding="utf-8"?>
                                <!DOCTYPE Map>
                                <Map bgcolor="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +over +no_defs">
                                  <Style name="style">
                                    <Rule>
                                      <PolygonSymbolizer>
                                        <CssParameter name="fill">#000000</CssParameter>
                                      </PolygonSymbolizer>
                                      <LineSymbolizer>
                                        <CssParameter name="stroke">#000000</CssParameter>
                                        <CssParameter name="stroke-width">0.0</CssParameter>
                                      </LineSymbolizer>
                                    </Rule>
                                  </Style>"""
                mapfile = str(app_globals.ECOSHP_DIR +'/'+ name + '.mapfile.xml')

                logging.info('Creating mapfile: %s' % (mapfile))

                pixels = 256

                res = (2 * math.pi * 6378137 / pixels) / (2**int(z))
                sh = 2 * math.pi * 6378137 / 2.0

                gy = (2**float(z) - 1) - int(y)
                minx, miny = ((float(x)*pixels) * res - sh),      (((float(gy))*pixels) * res - sh)
                maxx, maxy = (((float(x)+1)*pixels) * res - sh),  (((float(gy)+1)*pixels) * res - sh)

                minx, maxx = (minx / sh) * 180.0, (maxx / sh) * 180.0
                miny, maxy = (miny / sh) * 180.0, (maxy / sh) * 180.0
                miny = 180 / math.pi * (2 * math.atan( math.exp( miny * math.pi / 180.0)) - math.pi / 2.0)
                maxy = 180 / math.pi * (2 * math.atan( math.exp( maxy * math.pi / 180.0)) - math.pi / 2.0)

                bbox = (minx, miny, maxx, maxy)

                logging.info('Tiling %s with bbox: %s' % (name,str(bbox)))

                ct=0
                for id in region_ids:
                    shp = os.path.join(app_globals.ECOSHP_DIR, id + '.shp')
                    if os.path.exists(shp):
                        ct+=1
                        tmp_xml += """
                                <Layer name="layer_name" srs="+proj=latlong +datum=WGS84">
                                <StyleName>style</StyleName>
                                <Datasource>
                                  <Parameter name="type">shape</Parameter>
                                  <Parameter name="file">layer_name</Parameter>
                                </Datasource>
                              </Layer>""".replace("layer_name",id)

                tmp_xml += "</Map>"
                if ct>0:
                    tile_dir =  str(app_globals.ECOTILE_DIR.rstrip('/') + "/" + name +"/")
                    if newmapfile or not os.path.exists(mapfile):
                        open(mapfile, "w+").write(tmp_xml)
                    if not os.path.isdir(tile_dir):
                        os.mkdir(tile_dir)
                    GenerateTiles.render_tiles(bbox,
                                               mapfile,
                                               tile_dir,
                                               int(z),
                                               int(z),
                                               name,
                                               num_threads=app_globals.TILE_QUEUE_THREADS,
                                               overwrite=overwrite)
            if os.path.exists(png):
                logging.error('Returning tile : ' + png)
                response.headers['Content-Type'] = 'image/png'
                response.status = 200
                return open(png, 'rb').read()

            logging.info('Tile not found : ' + png)
            response.status = 404

        elif str(method).lower() == 'tilearea':
            """ Takes one to many record_ids split by commas and a unique
                name for the tileset you are creating.

                e.g., If you are creating a tileset for a single ecoregion
                with code NT105, you would probably want to create a new
                tileset with,
                    name = 'NT105',
                    record_ids = 'NT105'
                If you are creating a new tileset for hyp. spec 'Andreus
                andreus' you could create a tileset with,
                    name = 'animalia-mammalia-Andreus_andreus'
                    record_ids = 'NT103','NT445','NT6742'
            """

            try:
                region_ids = request.GET['region_ids'].split(',')
            except:
                region_ids = [name]
            lowx = request.GET['lowx']
            lowy = request.GET['lowy']
            highx = request.GET['highx']
            highy = request.GET['highy']
            zoom = int(request.GET['zoom'])

            request_ip = request.environ['REMOTE_ADDR']
            if request_ip == "127.0.0.1":
                tasks.EcoregionProcessingThread.apply_async(args=[name, zoom, lowx, lowy, highx, highy, region_ids])
                logging.info('Sending job to Celery')
                response.status = 200
                return
            else:
                logging.info("request from: %s" %request_ip)
                tile_dir =  str(app_globals.ECOTILE_DIR.rstrip('/') + "/" + name +"/")

                tmp_xml = """<?xml version="1.0" encoding="utf-8"?>
                                <!DOCTYPE Map>
                                <Map bgcolor="transparent" srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +over +no_defs">
                                  <Style name="style">
                                    <Rule>
                                      <PolygonSymbolizer>
                                        <CssParameter name="fill">#000000</CssParameter>
                                      </PolygonSymbolizer>
                                      <LineSymbolizer>
                                        <CssParameter name="stroke">#000000</CssParameter>
                                        <CssParameter name="stroke-width">0.0</CssParameter>
                                      </LineSymbolizer>
                                    </Rule>
                                  </Style>"""
                mapfile = str(app_globals.ECOSHP_DIR +'/'+ name + '.mapfile.xml')

                logging.info('Creating mapfile: %s' % (mapfile))

                bbox = (float(lowx), float(lowy), float(highx), float(highy))
                logging.info('Tiling %s with bbox: %s' % (name,str(bbox)))

                ct=0
                for id in region_ids:
                    shp = os.path.join(app_globals.ECOSHP_DIR, id + '.shp')
                    if os.path.exists(shp):
                        ct+=1
                        tmp_xml += """
                                <Layer name="layer_name" srs="+proj=latlong +datum=WGS84">
                                <StyleName>style</StyleName>
                                <Datasource>
                                  <Parameter name="type">shape</Parameter>
                                  <Parameter name="file">layer_name</Parameter>
                                </Datasource>
                              </Layer>""".replace("layer_name",id)

                tmp_xml += "</Map>"
                if ct>0:
                    open(mapfile, "w+").write(tmp_xml)
                    if not os.path.isdir(tile_dir):
                        os.mkdir(tile_dir)
                    GenerateTiles.render_tiles(bbox,
                                               mapfile,
                                               tile_dir,
                                               zoom,
                                               zoom,
                                               name,
                                               num_threads=app_globals.TILE_QUEUE_THREADS,
                                               overwrite=False)
                    response.status = 200
                    return

                logging.info('Regions not found : ' + shp)
                response.status = 404
<|endoftext|>


<commit_before>import curses
from threading import Thread
from time import sleep

from tsparser.utils import Singleton, StatisticDataCollector

class UserInterface(metaclass=Singleton):
    """
    User Interface is a singleton. Once ran, it renders UI until exiting the app.
    """

    def __init__(self, refreshing_frequency=1):
        self.__REFRESHING_FREQUENCY = refreshing_frequency

    def run(self):
        Thread(target=self.__renderer_thread, daemon=True).start()
        Thread(target=self.__event_handler_thread, daemon=True).start()

    def __event_handler_thread(self):
        # actions:
        # - exit (curses.endwin())
        # - filter modules
        # - scroll logs
        # - enable/disable auto scroll
        pass

    def __renderer_thread(self):
        self.__init_curses()
        while True:
            sleep(1/self.__REFRESHING_FREQUENCY)
            self.__render_frame()

    def __init_curses(self):
        self.__screen = curses.initscr()
        curses.start_color()
        curses.curs_set(0)

        self.__TIMESTAMP_COLOR = 1
        curses.init_pair(self.__TIMESTAMP_COLOR, curses.COLOR_RED, curses.COLOR_BLACK)
        self.__MODULE_NAME_COLOR = 2
        curses.init_pair(self.__MODULE_NAME_COLOR, curses.COLOR_BLUE, curses.COLOR_BLACK)
        self.__INFO_BAR_DESC_COLOR = 3
        curses.init_pair(self.__INFO_BAR_DESC_COLOR, curses.COLOR_BLACK, curses.COLOR_CYAN)

        self.__logs_auto_scrolling = True
        StatisticDataCollector().get_logger().log('system', 'User interface initialized!')

    def __render_frame(self):
        lines, cols = self.__screen.getmaxyx()
        if lines < 25 or cols < 80:
            self.__screen.clear()
            self.__screen.addstr('Terminal size should be at least 80x25!\n')
            self.__screen.refresh()
            return
        statistics_window_width = 40

        logs_windows = curses.newwin(lines - 1, cols - statistics_window_width, 0, 0)
        self.__render_logs(logs_windows)
        statistics_window = curses.newwin(lines - 1, statistics_window_width, 0, cols - statistics_window_width)
        self.__render_statistics(statistics_window)
        info_bar_window = curses.newwin(1, cols, lines - 1, 0)
        self.__render_info_bar(info_bar_window)

    def __render_logs(self, window):
        self.__draw_entitled_box(window, 'Logs')
        window.refresh()

        lines, cols = window.getmaxyx()
        pad = curses.newpad(lines - 2, cols - 2)
        logs = StatisticDataCollector().get_logger().get_logs()
        for timestamp, module_name, message in logs:
            # TODO do logs filtering here! (continue if module_name not in filter list)
            timestamp_str = '{:02}:{:02}:{:02}.{:06}'.format(timestamp.hour, timestamp.minute,
                                                             timestamp.second, timestamp.microsecond)
            module_name = module_name.replace('\n', '<nl>')
            message = message.replace('\n', '<nl>')

            # resizing pad if needed
            pad_lines, pad_cols = pad.getmaxyx()
            whole_message = timestamp_str + ' ' + module_name + ' ' + message + '\n'
            lines_needed = len(whole_message) // pad_cols + 1
            cursor_y, cursor_x = pad.getyx()
            lines_left = pad_lines - cursor_y
            if lines_left < lines_needed + 1:
                pad.resize((pad_lines + lines_needed) * 2, pad_cols)
                pad.move(cursor_y, cursor_x)

            # render line
            pad.addstr(timestamp_str+' ', curses.color_pair(self.__TIMESTAMP_COLOR))
            pad.addstr(module_name+' ', curses.color_pair(self.__MODULE_NAME_COLOR))
            pad.addstr(message)
            if len(whole_message) % pad_cols != 1:
                pad.addstr('\n')

        # show logs, TODO do scrolling here (currently implemented auto-scrolling only)
        pad_cursor_y, _ = pad.getyx()
        pad_visible_lines = lines - 2
        pad_visible_cols = cols - 2
        auto_scroll_position = max(0, pad_cursor_y - pad_visible_lines)
        beg_y, beg_x = window.getbegyx()
        pad.refresh(auto_scroll_position, 0, beg_y + 1, beg_x + 1, beg_y + pad_visible_lines, beg_x + pad_visible_cols)

        #test-purposes only - create logs
        import random
        StatisticDataCollector().get_logger().log('test', 'x'*random.randint(50, 52)+'y')

    def __render_statistics(self, window):
        self.__draw_entitled_box(window, 'Statistics')
        lines, cols = window.getmaxyx()
        beg_y, beg_x = window.getbegyx()
        sub_win = window.subwin(lines - 2, cols - 2, beg_y + 1, beg_x + 1)
        stats_to_display = self.__prepare_stats()
        for name, value in stats_to_display:
            sub_win.addstr('{}: {}\n'.format(name, value))
        window.refresh()

    @staticmethod
    def __prepare_stats():
        def timedelta_to_str(timedelta_obj):
            seconds = timedelta_obj.total_seconds()
            if seconds < 1:
                return '<1 sec'
            seconds = int(seconds)
            result = '{} sec'.format(seconds % 60)
            if seconds >= 60:
                result = '{} min {}'.format(seconds // 60, result)
            return result

        def data_amount_to_str(data_amount):
            steps = (
                (1, 'B'),
                (10**3, 'kB'),
                (10**6, 'MB'),
                (10**9, 'GB')
            )
            last_good_result = '0 B'
            for bound, unit in steps:
                if data_amount < bound:
                    return last_good_result
                last_good_result = '{:.3f} {}'.format(data_amount / bound, unit)
            return last_good_result

        sdc = StatisticDataCollector()
        stats_scheme = (
            ('Time since last receiving', timedelta_to_str(sdc.get_time_since_last_data_receiving())),
            ('Time since start', timedelta_to_str(sdc.get_time_since_start())),
            ('Data receiving speed', data_amount_to_str(sdc.get_data_receiving_speed())+'/s'),
            ('Total data received', data_amount_to_str(sdc.get_total_data_received())),
            ('Queued requests', str(sdc.get_count_of_queued_requests())),
            ('Total sent requests', str(sdc.get_total_count_of_sent_requests()))
        )
        return stats_scheme

    @staticmethod
    def __draw_entitled_box(window, title):
        window.box()
        window.addstr(0, 1, title)

    def __render_info_bar(self, window):
        info_bar_scheme = (
            ('F1', '{} auto scrolling'.format('Disable' if self.__logs_auto_scrolling else 'Enable')),
            ('F2', 'Filter'),
            ('F3', 'Exit'),
            ('↑↓', 'Scroll')
        )
        for key, description in info_bar_scheme:
            window.addstr(key)
            window.addstr(description, curses.color_pair(self.__INFO_BAR_DESC_COLOR))
        window.refresh()
<commit_msg>Partially implemented actions for UI<commit_after>import curses
import os
from threading import Thread
from time import sleep

from tsparser.utils import Singleton, StatisticDataCollector

class UserInterface(metaclass=Singleton):
    """
    User Interface is a singleton. Once ran, it renders UI until exiting the app.
    """

    def __init__(self, refreshing_frequency=1):
        self.__REFRESHING_FREQUENCY = refreshing_frequency

    def run(self):
        Thread(target=self.__interface_thread, daemon=True).start()

    def __interface_thread(self):
        self.__init_curses()
        while True:
            sleep(1/self.__REFRESHING_FREQUENCY)
            self.__process_events()
            self.__render_frame()

    def __init_curses(self):
        self.__screen = curses.initscr()
        curses.start_color()
        curses.curs_set(0)
        self.__screen.nodelay(True)
        self.__screen.keypad(True)

        self.__TIMESTAMP_COLOR = 1
        curses.init_pair(self.__TIMESTAMP_COLOR, curses.COLOR_RED, curses.COLOR_BLACK)
        self.__MODULE_NAME_COLOR = 2
        curses.init_pair(self.__MODULE_NAME_COLOR, curses.COLOR_BLUE, curses.COLOR_BLACK)
        self.__INFO_BAR_DESC_COLOR = 3
        curses.init_pair(self.__INFO_BAR_DESC_COLOR, curses.COLOR_BLACK, curses.COLOR_CYAN)
        self.__FILTER_WINDOW_BACKGROUND = 4
        curses.init_pair(self.__FILTER_WINDOW_BACKGROUND, curses.COLOR_WHITE, curses.COLOR_BLUE)
        self.__FILTER_WINDOW_SELECTION = 5
        curses.init_pair(self.__FILTER_WINDOW_SELECTION, curses.COLOR_BLACK, curses.COLOR_CYAN)

        self.__logs_auto_scrolling = True
        self.__scroll_position = self.__auto_scroll_position = 1
        self.__filter_window_active = False
        StatisticDataCollector().get_logger().log('system', 'User interface initialized!')

    def __process_events(self):
        while True:
            key_code = self.__screen.getch()
            if key_code == curses.ERR:
                return
            StatisticDataCollector().get_logger().log('ui', 'got key code: {}'.format(key_code))

            if self.__filter_window_active:
                self.__filter_window_process_event(key_code)
            else:
                self.__main_window_process_event(key_code)

    def __filter_window_process_event(self, key_code):
        if key_code == 27:  # escape
            self.__filter_window_active = False
        elif key_code == ord(' '):
            pass
        elif key_code == curses.KEY_UP:
            pass
        elif key_code == curses.KEY_DOWN:
            pass

    def __main_window_process_event(self, key_code):
        if key_code == curses.KEY_F2:
            self.__logs_auto_scrolling = not self.__logs_auto_scrolling
        elif key_code == curses.KEY_F3:
            self.__filter_window_active = True
        elif key_code == curses.KEY_F9:
            curses.endwin()
            os.kill(os.getpid(), 15)
        elif key_code == curses.KEY_UP:
            self.__logs_auto_scrolling = False
            self.__scroll_position = max(self.__scroll_position - 1, 0)
        elif key_code == curses.KEY_DOWN:
            if self.__scroll_position == self.__auto_scroll_position:
                self.__logs_auto_scrolling = True
            else:
                self.__scroll_position += 1

    def __render_frame(self):
        lines, cols = self.__screen.getmaxyx()
        if lines < 25 or cols < 80:
            self.__screen.clear()
            self.__screen.addstr('Terminal size should be at least 80x25!\n')
            self.__screen.refresh()
            return
        statistics_window_width = 40

        logs_windows = curses.newwin(lines - 1, cols - statistics_window_width, 0, 0)
        self.__render_logs(logs_windows)
        statistics_window = curses.newwin(lines - 1, statistics_window_width, 0, cols - statistics_window_width)
        self.__render_statistics(statistics_window)
        info_bar_window = curses.newwin(1, cols, lines - 1, 0)
        self.__render_info_bar(info_bar_window)
        if self.__filter_window_active:
            width, height = 60, 20
            filter_window = curses.newwin(height, width, (lines - height) // 2, (cols - width) // 2)
            self.__render_filter_window(filter_window)

    def __render_logs(self, window):
        self.__draw_entitled_box(window, 'Logs')
        window.refresh()

        lines, cols = window.getmaxyx()
        pad = curses.newpad(lines - 2, cols - 2)
        logs = StatisticDataCollector().get_logger().get_logs()
        for timestamp, module_name, message in logs:
            # TODO do logs filtering here! (continue if module_name not in filter list)
            # TODO render only what is visible!
            # TODO write this section from scratch! (remember about scroll position and filters!)
            timestamp_str = '{:02}:{:02}:{:02}.{:06}'.format(timestamp.hour, timestamp.minute,
                                                             timestamp.second, timestamp.microsecond)
            module_name = module_name.replace('\n', '<nl>')
            message = message.replace('\n', '<nl>')

            # resizing pad if needed
            pad_lines, pad_cols = pad.getmaxyx()
            whole_message = timestamp_str + ' ' + module_name + ' ' + message + '\n'
            lines_needed = len(whole_message) // pad_cols + 1
            cursor_y, cursor_x = pad.getyx()
            lines_left = pad_lines - cursor_y
            if lines_left < lines_needed + 1:
                pad.resize((pad_lines + lines_needed) * 2, pad_cols)
                pad.move(cursor_y, cursor_x)

            # render line
            pad.addstr(timestamp_str+' ', curses.color_pair(self.__TIMESTAMP_COLOR))
            pad.addstr(module_name+' ', curses.color_pair(self.__MODULE_NAME_COLOR))
            pad.addstr(message)
            if len(whole_message) % pad_cols != 1:
                pad.addstr('\n')

        # show logs
        pad_cursor_y, _ = pad.getyx()
        pad_visible_lines = lines - 2
        pad_visible_cols = cols - 2
        self.__auto_scroll_position = pad_cursor_y
        if self.__logs_auto_scrolling:
            self.__scroll_position = self.__auto_scroll_position
        beg_y, beg_x = window.getbegyx()
        pad.refresh(max(0, self.__scroll_position - pad_visible_lines), 0,
                    beg_y + 1, beg_x + 1, beg_y + pad_visible_lines, beg_x + pad_visible_cols)

        #test-purposes only - create logs # TODO remove it
        import random
        StatisticDataCollector().get_logger().log('test', 'x'*random.randint(50, 52)+'y')

    def __render_statistics(self, window):
        self.__draw_entitled_box(window, 'Statistics')
        sub_win = self.__get_sub_window(window)
        stats_to_display = self.__prepare_stats()
        for name, value in stats_to_display:
            sub_win.addstr('{}: {}\n'.format(name, value))
        window.refresh()

    @staticmethod
    def __prepare_stats():
        def timedelta_to_str(timedelta_obj):
            seconds = timedelta_obj.total_seconds()
            if seconds < 1:
                return '<1 sec'
            seconds = int(seconds)
            result = '{} sec'.format(seconds % 60)
            if seconds >= 60:
                result = '{} min {}'.format(seconds // 60, result)
            return result

        def data_amount_to_str(data_amount):
            steps = (
                (1, 'B'),
                (10**3, 'kB'),
                (10**6, 'MB'),
                (10**9, 'GB')
            )
            last_good_result = '0 B'
            for bound, unit in steps:
                if data_amount < bound:
                    return last_good_result
                last_good_result = '{:.3f} {}'.format(data_amount / bound, unit)
            return last_good_result

        sdc = StatisticDataCollector()
        stats_scheme = (
            ('Time since last receiving', timedelta_to_str(sdc.get_time_since_last_data_receiving())),
            ('Time since start', timedelta_to_str(sdc.get_time_since_start())),
            ('Data receiving speed', data_amount_to_str(sdc.get_data_receiving_speed())+'/s'),
            ('Total data received', data_amount_to_str(sdc.get_total_data_received())),
            ('Queued requests', str(sdc.get_count_of_queued_requests())),
            ('Total sent requests', str(sdc.get_total_count_of_sent_requests()))
        )
        return stats_scheme

    @staticmethod
    def __draw_entitled_box(window, title):
        window.box()
        window.addstr(0, 1, title)

    @staticmethod
    def __get_sub_window(window, margin=1):
        lines, cols = window.getmaxyx()
        beg_y, beg_x = window.getbegyx()
        return window.subwin(lines - margin * 2, cols - margin * 2, beg_y + margin, beg_x + margin)

    def __render_info_bar(self, window):
        info_bar_scheme = (
            ('F2', '{} auto scrolling'.format('Disable' if self.__logs_auto_scrolling else 'Enable')),
            ('F3', 'Filter'),
            ('F9', 'Exit'),
            ('↑↓', 'Scroll')
        )
        for key, description in info_bar_scheme:
            window.addstr(key)
            window.addstr(description, curses.color_pair(self.__INFO_BAR_DESC_COLOR))
        window.refresh()

    def __render_filter_window(self, window):
        window.bkgd(' ', curses.color_pair(self.__FILTER_WINDOW_BACKGROUND))
        self.__draw_entitled_box(window, 'Filter')
        sub_win = self.__get_sub_window(window)
        sub_win.addstr('Please use arrows, space and escape to navigate.\n')
        # TODO implement filtering window logic
        window.refresh()
<|endoftext|>


<commit_before>from ui import *

################################################################################
# example usage

txt1 = Text(('hello world 1\n' * 2)[:-1])
txt2 = TextNoWrap(((('hello world 2 ' * 10) + '\n') * 10)[:-1])
b1 = Box(txt1, title_parts=[' plack ', ' dag '], option_parts=[' blah '])
b2 = Box(txt2, title_parts=[' doop '], option_parts=[' example ', ' auh '])
p = VerticalPile(b1, b2, b2, b2, b2)

class App(Application):
    windows = {'default': p}

    def handle(self, key):
        if key == 'a':
            pass
        super(App, self).handle(key)

if __name__ == '__main__':
    App().run()
<commit_msg>Change the example to more closely mirror the booger interface<commit_after>from ui import *

################################################################################
# example usage

status = Text('Tests something | ok: 0 | error: 0 | fail: 0', style='RB')
try:
    aoeu
except Exception as e:
    err = str(e)
exception = TextNoWrap(err)
test = Box(exception,
           title_parts=[' F ', ' a_test '],
           option_parts=[' Traceback ', ' stdOut '])

p = VerticalPile(status, test)

class App(Application):
    windows = {'default': p}

    def handle(self, key):
        if key == 'a':
            pass
        super(App, self).handle(key)

if __name__ == '__main__':
    App().run()
<|endoftext|>


<commit_before>from services import ServiceHost, ServiceClient, api, publish, request, subscribe

class IdentityService(ServiceHost):
    @api
    def create(self, user_name, password, request_id=None, entity=None):
        return True

    @publish
    def password_changed(self, user_name):
        pass

class IdentityClient(ServiceClient):
    @request
    def create(self, user_name, password):
        return locals()

    @subscribe
    def password_changed(self, user_name):
        pass<commit_msg>added a simple description in example.py<commit_after>from services import ServiceHost, ServiceClient, api, publish, request, subscribe

class IdentityService(ServiceHost):
    @api
    def create(self, user_name, password, request_id=None, entity=None):
        """ This method is called by the bus when a client remotely calls this api """
        return True

    @publish
    def password_changed(self, user_name):
        """Calling this method causes the password_changed message to be sent on the bus"""
        return user_name

class IdentityClient(ServiceClient):
    @request
    def create(self, user_name, password):
        """Calling this method remotely invokes this api, must be empty and return locals"""
        return locals()

    @subscribe
    def password_changed(self, user_name):
        """This method is called by the bus when a ServiceHost publishes something"""
        pass

<|endoftext|>


<commit_before># Copyright (c) 2012, 2013 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
"""
Interface to library of synthesis tools, e.g., JTLV, gr1c
"""
from copy import deepcopy

from tulip import transys
from tulip.spec import GRSpec
from tulip import jtlvint
from tulip import gr1cint

def _disj(set0):
    return " || ".join([
        "(" +str(x) +")"
        for x in set0
    ])

def _conj_intersection(set0, set1, parenth=True):
    if parenth:
        return " && ".join([
            "("+str(x)+")"
            for x in set0
            if x in set1
        ])
    else:
        return " && ".join([
            str(x)
            for x in set0
            if x in set1
        ])

def _conj_neg(set0, parenth=True):
    if parenth:
        return " && ".join([
            "!("+str(x)+")"
            for x in set0
        ])
    else:
        return " && ".join([
            "!"+str(x)
            for x in set0
        ])

def _conj_neg_diff(set0, set1, parenth=True):
    if parenth:
        return " && ".join([
            "!("+str(x)+")"
            for x in set0
            if x not in set1
        ])
    else:
        return " && ".join([
            "!"+str(x)
            for x in set0
            if x not in set1
        ])

def sys_to_spec(sys):
    """Convert finite transition system to GR(1) representation.

    The term GR(1) representation is preferred to GR(1) spec,
    because an FTS can represent sys_init, sys_safety, but
    not other spec forms.

    @type sys: transys.FiniteTransitionSystem

    @rtype: GRSpec
    """
    if isinstance(sys, transys.FiniteTransitionSystem):
        return fts2spec(sys)
    elif isinstance(sys, transys.OpenFiniteTransitionSystem):
        return open_fts2spec(sys)
    else:
        raise TypeError('synth.sys_to_spec does not support ' +
            str(type(sys)) +'. Use FTS or OpenFTS.')

def fts2spec(fts):
    """Convert closed FTS to GR(1) representation.

    So fts on its own is not the complete problem spec.

    @param fts: transys.FiniteTransitionSystem

    @rtype: GRSpec
    """
    assert(isinstance(fts, transys.FiniteTransitionSystem))

    aps = fts.aps
    states = fts.states

    # everything is controlled
    sys_vars = list(aps)
    sys_vars.extend([s for s in states])

    init = sys_init_from_ts(states, aps)

    trans = sys_trans_from_ts(states)
    trans += ap_trans_from_ts(states, aps)
    trans += sys_state_mutex(states)

    return GRSpec(sys_vars=sys_vars, sys_init=init, sys_safety=trans)

def open_fts2spec(ofts):
    """Convert OpenFTS to GR(1) representation.

    Note that not any GR(1) can be represented by an OpenFTS,
    as the OpenFTS is currently defined.
    A GameStructure would be needed instead.

    Use the spec to add more information,
    for example to specify env_init, sys_init that
    involve both sys_vars and env_vars.

    For example, an OpenFTS cannot represent how the
    initial valuation of env_vars affects the allowable
    initial valuation of sys_vars, which is represented
    by the state of OpenFTS.

    Either OpenFTS can be extended in the future,
    or a game structure added.

    notes
    -----

    1. Currently each env_action becomes a bool env_var.
        In the future a candidate option is to represent the
        env_actions as an enumeration.
        This would avoid the exponential cost incurred by bools.
        A map between the enum and named env_actions
        will probably also be needed.

    @para ofts: transys.OpenFiniteTransitionSystem

    @rtype: GRSpec
    """
    assert(isinstance(ofts, transys.OpenFiniteTransitionSystem))

    aps = ofts.aps
    states = ofts.states
    trans = ofts.transitions

    sys_vars = list(aps)
    sys_vars.extend([s for s in states])

    env_vars = list(ofts.env_actions)

    sys_init = sys_init_from_ts(states, aps)

    sys_trans = sys_trans_from_open_ts(states, trans, env_vars)
    sys_trans += ap_trans_from_ts(states, aps)
    sys_trans += sys_state_mutex(states)

    env_trans = env_trans_from_open_ts(states, trans, env_vars)
    env_trans += sys_state_mutex(env_vars)

    return GRSpec(
        sys_vars=sys_vars, env_vars=env_vars,
        sys_init=sys_init,
        env_safety=env_trans,
        sys_safety=sys_trans
    )

def sys_init_from_ts(states, aps):
    """Initial state, including enforcement of mutual exclusion.
    """
    if (len(states.initial) > 0):
        init = [_disj([
            "("+str(x)+")" +" && " +_conj_neg_diff(states, [x])
            for x in states.initial
        ])]
    else:
        init = ""

    for state in states.initial:
        label = states.label_of(state)

        ap_init = _conj_intersection(aps, label["ap"])
        init.append(ap_init)

        if len(init[-1]) > 0:
            init[-1] += " && "

        init[-1] += _conj_neg_diff(aps, label["ap"])
        init[-1] = "("+str(state)+") -> ("+init[-1]+")"
    return init

def sys_trans_from_ts(states):
    """Convert environment actions to GR(1) representation.
    """
    trans = []

    for from_state in states:
        post = states.post(from_state)

        # no successor states ?
        if not post:
            continue

        post_states = _disj(post)
        trans += ["(" +str(from_state) +") -> X(" +post_states +")"]
    return trans

def sys_state_mutex(states):
    """Mutual exclusion of states.
    """
    trans = [
        "X("+_disj([
            "("+str(x)+")"+" && " +_conj_neg_diff(states, [x])
            for x in states
        ])+")"
    ]
    return trans

def sys_trans_from_open_ts(states, trans, env_vars):
    """Convert sys transitions and env actions to GR(1) sys_safety.

    Mutexes not enforced by this function:

        - among sys states
        - among env actions
    """
    sys_trans = []

    # Transitions
    for from_state in states:
        cur_trans = trans.find([from_state])

        # no successor states ?
        if not cur_trans:
            continue

        for (from_state, to_state, label) in cur_trans:
            env_action = label['env_actions']

            precond = "(" +str(from_state) +")"
            precond = precond +" && X(" + str(env_action) + ")"
            sys_trans += ["(" + precond + ") -> X(" +str(to_state) +")"]
    return sys_trans

def env_trans_from_open_ts(states, trans, env_vars):
    """Convert environment actions to GR(1) env_safety.
    """
    env_trans = []

    for from_state in states:
        cur_trans = trans.find([from_state])

        # no successor states ?
        if not cur_trans:
            continue

        # collect possible next env actions
        next_env_actions = set()
        for (from_state, to_state, label) in cur_trans:
            env_action = label['env_actions']
            next_env_actions.add(env_action)
        next_env_actions = _disj(next_env_actions)

        env_trans += ["(" +str(from_state) +") -> X("+ next_env_actions +")"]
    return env_trans

def ap_trans_from_ts(states, aps):
    """Require atomic propositions to follow states according to label.
    """
    trans = []

    for state in states:
        label = states.label_of(state)

        if label.has_key("ap"):
            tmp = _conj_intersection(aps, label["ap"], parenth=False)
        else:
            tmp = ""

        if len(tmp) > 0:
            tmp += " && "

        if label.has_key("ap"):
            tmp += _conj_neg_diff(aps, label["ap"], parenth=False)
        else:
            tmp += _conj_neg(aps, parenth=False)

        trans += ["X(("+ str(state) +") -> ("+ tmp +"))"]
    return trans

def synthesize(option, specs, sys=None, verbose=0):
    """Function to call the appropriate synthesis tool on the spec.

    Beware!  This function provides a generic interface to a variety
    of routines.  Being under active development, the types of
    arguments supported and types of objects returned may change
    without notice.

    @param option: Magic string that declares what tool to invoke,
        what method to use, etc.  Currently recognized forms:

          - C{"gr1c"}: use gr1c for GR(1) synthesis via L{gr1cint}.
          - C{"jtlv"}: use JTLV for GR(1) synthesis via L{jtlvint}.
    @type specs: L{spec.GRSpec}
    @param sys: A transition system that should be expressed with the
        specification (spec).

    @type verbose: bool

    @return: If spec is realizable,
        then return a Mealy machine implementing the strategy.
        Otherwise return list of counterexamples.
    @rtype: transys.Mealy or list
    """
    if sys is not None:
        sys = deepcopy(sys)

        sform = sys_to_spec(sys)
        specs = specs | sform

    if option == 'gr1c':
        ctrl = gr1cint.synthesize(specs, verbose=verbose)
    elif option == 'jtlv':
        ctrl = jtlvint.synthesize(specs, verbose=verbose)
    else:
        raise Exception('Undefined synthesis option. '+\
                        'Current options are "jtlv" and "gr1c"')
    return ctrl
<commit_msg>new function: synth.is_realizable<commit_after># Copyright (c) 2012, 2013 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
"""
Interface to library of synthesis tools, e.g., JTLV, gr1c
"""
from copy import deepcopy

from tulip import transys
from tulip.spec import GRSpec
from tulip import jtlvint
from tulip import gr1cint

def _disj(set0):
    return " || ".join([
        "(" +str(x) +")"
        for x in set0
    ])

def _conj_intersection(set0, set1, parenth=True):
    if parenth:
        return " && ".join([
            "("+str(x)+")"
            for x in set0
            if x in set1
        ])
    else:
        return " && ".join([
            str(x)
            for x in set0
            if x in set1
        ])

def _conj_neg(set0, parenth=True):
    if parenth:
        return " && ".join([
            "!("+str(x)+")"
            for x in set0
        ])
    else:
        return " && ".join([
            "!"+str(x)
            for x in set0
        ])

def _conj_neg_diff(set0, set1, parenth=True):
    if parenth:
        return " && ".join([
            "!("+str(x)+")"
            for x in set0
            if x not in set1
        ])
    else:
        return " && ".join([
            "!"+str(x)
            for x in set0
            if x not in set1
        ])

def sys_to_spec(sys):
    """Convert finite transition system to GR(1) representation.

    The term GR(1) representation is preferred to GR(1) spec,
    because an FTS can represent sys_init, sys_safety, but
    not other spec forms.

    @type sys: transys.FiniteTransitionSystem

    @rtype: GRSpec
    """
    if isinstance(sys, transys.FiniteTransitionSystem):
        return fts2spec(sys)
    elif isinstance(sys, transys.OpenFiniteTransitionSystem):
        return open_fts2spec(sys)
    else:
        raise TypeError('synth.sys_to_spec does not support ' +
            str(type(sys)) +'. Use FTS or OpenFTS.')

def fts2spec(fts):
    """Convert closed FTS to GR(1) representation.

    So fts on its own is not the complete problem spec.

    @param fts: transys.FiniteTransitionSystem

    @rtype: GRSpec
    """
    assert(isinstance(fts, transys.FiniteTransitionSystem))

    aps = fts.aps
    states = fts.states

    # everything is controlled
    sys_vars = list(aps)
    sys_vars.extend([s for s in states])

    init = sys_init_from_ts(states, aps)

    trans = sys_trans_from_ts(states)
    trans += ap_trans_from_ts(states, aps)
    trans += sys_state_mutex(states)

    return GRSpec(sys_vars=sys_vars, sys_init=init, sys_safety=trans)

def open_fts2spec(ofts):
    """Convert OpenFTS to GR(1) representation.

    Note that not any GR(1) can be represented by an OpenFTS,
    as the OpenFTS is currently defined.
    A GameStructure would be needed instead.

    Use the spec to add more information,
    for example to specify env_init, sys_init that
    involve both sys_vars and env_vars.

    For example, an OpenFTS cannot represent how the
    initial valuation of env_vars affects the allowable
    initial valuation of sys_vars, which is represented
    by the state of OpenFTS.

    Either OpenFTS can be extended in the future,
    or a game structure added.

    notes
    -----

    1. Currently each env_action becomes a bool env_var.
        In the future a candidate option is to represent the
        env_actions as an enumeration.
        This would avoid the exponential cost incurred by bools.
        A map between the enum and named env_actions
        will probably also be needed.

    @para ofts: transys.OpenFiniteTransitionSystem

    @rtype: GRSpec
    """
    assert(isinstance(ofts, transys.OpenFiniteTransitionSystem))

    aps = ofts.aps
    states = ofts.states
    trans = ofts.transitions

    sys_vars = list(aps)
    sys_vars.extend([s for s in states])

    env_vars = list(ofts.env_actions)

    sys_init = sys_init_from_ts(states, aps)

    sys_trans = sys_trans_from_open_ts(states, trans, env_vars)
    sys_trans += ap_trans_from_ts(states, aps)
    sys_trans += sys_state_mutex(states)

    env_trans = env_trans_from_open_ts(states, trans, env_vars)
    env_trans += sys_state_mutex(env_vars)

    return GRSpec(
        sys_vars=sys_vars, env_vars=env_vars,
        sys_init=sys_init,
        env_safety=env_trans,
        sys_safety=sys_trans
    )

def sys_init_from_ts(states, aps):
    """Initial state, including enforcement of mutual exclusion.
    """
    if (len(states.initial) > 0):
        init = [_disj([
            "("+str(x)+")" +" && " +_conj_neg_diff(states, [x])
            for x in states.initial
        ])]
    else:
        init = ""

    for state in states.initial:
        label = states.label_of(state)

        ap_init = _conj_intersection(aps, label["ap"])
        init.append(ap_init)

        if len(init[-1]) > 0:
            init[-1] += " && "

        init[-1] += _conj_neg_diff(aps, label["ap"])
        init[-1] = "("+str(state)+") -> ("+init[-1]+")"
    return init

def sys_trans_from_ts(states):
    """Convert environment actions to GR(1) representation.
    """
    trans = []

    for from_state in states:
        post = states.post(from_state)

        # no successor states ?
        if not post:
            continue

        post_states = _disj(post)
        trans += ["(" +str(from_state) +") -> X(" +post_states +")"]
    return trans

def sys_state_mutex(states):
    """Mutual exclusion of states.
    """
    trans = [
        "X("+_disj([
            "("+str(x)+")"+" && " +_conj_neg_diff(states, [x])
            for x in states
        ])+")"
    ]
    return trans

def sys_trans_from_open_ts(states, trans, env_vars):
    """Convert sys transitions and env actions to GR(1) sys_safety.

    Mutexes not enforced by this function:

        - among sys states
        - among env actions
    """
    sys_trans = []

    # Transitions
    for from_state in states:
        cur_trans = trans.find([from_state])

        # no successor states ?
        if not cur_trans:
            continue

        for (from_state, to_state, label) in cur_trans:
            env_action = label['env_actions']

            precond = "(" +str(from_state) +")"
            precond = precond +" && X(" + str(env_action) + ")"
            sys_trans += ["(" + precond + ") -> X(" +str(to_state) +")"]
    return sys_trans

def env_trans_from_open_ts(states, trans, env_vars):
    """Convert environment actions to GR(1) env_safety.
    """
    env_trans = []

    for from_state in states:
        cur_trans = trans.find([from_state])

        # no successor states ?
        if not cur_trans:
            continue

        # collect possible next env actions
        next_env_actions = set()
        for (from_state, to_state, label) in cur_trans:
            env_action = label['env_actions']
            next_env_actions.add(env_action)
        next_env_actions = _disj(next_env_actions)

        env_trans += ["(" +str(from_state) +") -> X("+ next_env_actions +")"]
    return env_trans

def ap_trans_from_ts(states, aps):
    """Require atomic propositions to follow states according to label.
    """
    trans = []

    for state in states:
        label = states.label_of(state)

        if label.has_key("ap"):
            tmp = _conj_intersection(aps, label["ap"], parenth=False)
        else:
            tmp = ""

        if len(tmp) > 0:
            tmp += " && "

        if label.has_key("ap"):
            tmp += _conj_neg_diff(aps, label["ap"], parenth=False)
        else:
            tmp += _conj_neg(aps, parenth=False)

        trans += ["X(("+ str(state) +") -> ("+ tmp +"))"]
    return trans

def synthesize(option, specs, sys=None, verbose=0):
    """Function to call the appropriate synthesis tool on the spec.

    Beware!  This function provides a generic interface to a variety
    of routines.  Being under active development, the types of
    arguments supported and types of objects returned may change
    without notice.

    @param option: Magic string that declares what tool to invoke,
        what method to use, etc.  Currently recognized forms:

          - C{"gr1c"}: use gr1c for GR(1) synthesis via L{gr1cint}.
          - C{"jtlv"}: use JTLV for GR(1) synthesis via L{jtlvint}.
    @type specs: L{spec.GRSpec}
    @param sys: A transition system that should be expressed with the
        specification (spec).

    @type verbose: bool

    @return: If spec is realizable,
        then return a Mealy machine implementing the strategy.
        Otherwise return list of counterexamples.
    @rtype: transys.Mealy or list
    """
    specs = spec_plus_sys(specs, sys)

    if option == 'gr1c':
        ctrl = gr1cint.synthesize(specs, verbose=verbose)
    elif option == 'jtlv':
        ctrl = jtlvint.synthesize(specs, verbose=verbose)
    else:
        raise Exception('Undefined synthesis option. '+\
                        'Current options are "jtlv" and "gr1c"')
    return ctrl

def is_realizable(option, specs, sys=None, verbose=0):
    """Check realizability.

    For details see synthesize.
    """
    specs = spec_plus_sys(specs, sys)

    if option == 'gr1c':
        r = gr1cint.check_realizable(specs, verbose=verbose)
    elif option == 'jtlv':
        r = jtlvint.check_realizable(specs, verbose=verbose)
    else:
        raise Exception('Undefined synthesis option. '+\
                        'Current options are "jtlv" and "gr1c"')
    return r

def spec_plus_sys(specs, sys=None):
    if sys is not None:
        sys = deepcopy(sys)

        sform = sys_to_spec(sys)
        specs = specs | sform
    return specs
<|endoftext|>


<commit_before>#!/usr/bin/env python

from harness import Game

game = Game(title="pysdl2 HARNESS demo", width=240, height=240, zoom=3)
background = game.load_resource("background.png")
title = game.load_resource("title.png")
font = game.load_bitmap_font("font.png", width=6, height=10)

scenes = []
hiscore = 0

class MenuScene(object):

    def __init__(self):
        self.counter = 2
        self.title_y = -100

    def draw(self, renderer):
        renderer.draw(background)
        renderer.draw(title, dest_rect=(0, int(self.title_y), 240, 60))

        # wait until the title is in place
        if self.title_y >= 40:
            if 2 < self.counter < 12:
                renderer.draw_text(font, 120, 120, "Press 's' to start!", align="center")

            renderer.draw_text(font, 120, 10, "Copyright (c) 2015 usebox.net", align="center")
            renderer.draw_text(font, 120, 48, "HI: %04i" % hiscore, align="center")

            renderer.draw_text(font, 120, 196, "Use the arrows to move", align="center")
            renderer.draw_text(font, 120, 208, "and collect the goodies!", align="center")

    def update(self, dt):

        if self.title_y < 40:
            self.title_y += dt * 120
            return

        self.counter += dt * 10
        if self.counter > 12:
            self.counter -= 12

        if game.keys[game.KEY_ESCAPE]:
            game.quit()
        elif game.keys[game.KEY_S]:
            # press "s" to play
            scenes.append(PlayScene())
            return

class PlayScene(object):

    def __init__(self):
        self.score = 0
        self.stage = 1
        self.time = 99

    def draw(self, renderer):
        renderer.draw(background)

        renderer.draw_text(font, 4, 4, "SCORE %04i" % self.score)
        renderer.draw_text(font, 236, 4, "STAGE %i" % self.stage, align="right")
        renderer.draw_text(font, 120, 9, "TIME: %02i" % int(self.time), align="center")

    def update(self, dt):
        self.time -= dt

        if game.keys[game.KEY_ESCAPE]:
            # avoid leaving the game just after
            # leaving this scene!
            game.keys[game.KEY_ESCAPE] = False

            # go to previous scene
            scenes.pop()
            return

@game.draw
def draw(renderer):
    # draw last scene
    scenes[-1].draw(renderer)

@game.update
def update(dt):
    # uddate last scene
    scenes[-1].update(dt)

scenes.append(MenuScene())

game.loop()

<commit_msg>about python version<commit_after>#!/usr/bin/env python
"""
Harness demo, a game example.

Tested with Python 3.4, may or may not work with Python 2!
"""

from harness import Game

game = Game(title="pysdl2 HARNESS demo", width=240, height=240, zoom=3)
background = game.load_resource("background.png")
title = game.load_resource("title.png")
font = game.load_bitmap_font("font.png", width=6, height=10)

scenes = []
hiscore = 0

class MenuScene(object):

    def __init__(self):
        self.counter = 2
        self.title_y = -100

    def draw(self, renderer):
        renderer.draw(background)
        renderer.draw(title, dest_rect=(0, int(self.title_y), 240, 60))

        # wait until the title is in place
        if self.title_y >= 40:
            if 2 < self.counter < 12:
                renderer.draw_text(font, 120, 120, "Press 's' to start!", align="center")

            renderer.draw_text(font, 120, 10, "Copyright (c) 2015 usebox.net", align="center")
            renderer.draw_text(font, 120, 48, "HI: %04i" % hiscore, align="center")

            renderer.draw_text(font, 120, 196, "Use the arrows to move", align="center")
            renderer.draw_text(font, 120, 208, "and collect the goodies!", align="center")

    def update(self, dt):

        if self.title_y < 40:
            self.title_y += dt * 120
            return

        self.counter += dt * 10
        if self.counter > 12:
            self.counter -= 12

        if game.keys[game.KEY_ESCAPE]:
            game.quit()
        elif game.keys[game.KEY_S]:
            # press "s" to play
            scenes.append(PlayScene())
            return

class PlayScene(object):

    def __init__(self):
        self.score = 0
        self.stage = 1
        self.time = 99

    def draw(self, renderer):
        renderer.draw(background)

        renderer.draw_text(font, 4, 4, "SCORE %04i" % self.score)
        renderer.draw_text(font, 236, 4, "STAGE %i" % self.stage, align="right")
        renderer.draw_text(font, 120, 9, "TIME: %02i" % int(self.time), align="center")

    def update(self, dt):
        self.time -= dt

        if game.keys[game.KEY_ESCAPE]:
            # avoid leaving the game just after
            # leaving this scene!
            game.keys[game.KEY_ESCAPE] = False

            # go to previous scene
            scenes.pop()
            return

@game.draw
def draw(renderer):
    # draw last scene
    scenes[-1].draw(renderer)

@game.update
def update(dt):
    # uddate last scene
    scenes[-1].update(dt)

scenes.append(MenuScene())

game.loop()

<|endoftext|>


<commit_before>import turtle
wn = turtle.Screen()
yertle = turtle.Turtle()

numSides = int (raw_input("How many sides will our regular polygon have?") )
sideLength = int (raw_input("What should the length of each side be?") )
color = raw_input("What color should we make the border of the polygon?")
fillColor = raw_input("What color should we fill the polygon with?")

yertle.color(color)
yertle.fillcolor(fillColor)
yertle.begin_fill()
extAngle = float(360)/numSides

for side in range(numSides):
    yertle.left(extAngle)
    yertle.forward(sideLength)

yertle.end_fill()
<commit_msg>Delete turtle_ex_6.py<commit_after><|endoftext|>


<commit_before>import logging
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import os
import re
import fnmatch
import App.OctLayers as Oct

def getLayerAsZScore(patient,controls,surface1,surface2):
    layer = patient.getThickness(surface1, surface2)
    layer_mean = controls.getLayerValues(surface1, surface2, 'mean')
    layer_sd = controls.getLayerValues(surface1, surface2, 'stdev')
    layer_zScore = (layer - layer_mean) / layer_sd
    return layer_zScore

def load_control_data(path, laterality):
    controls = Oct.OctCollection(folder = path,
                                 laterality = laterality)
    return controls

def plotLayerZscores(patient,controls,surface1,surface2,filename):
    colors = [(0,0,0),(1,0,0),(0,1,0)]
    cm = mpl.colors.ListedColormap(colors)
    thickness = getLayerAsZScore(patient, controls, surface1, surface2)
    thickness_map = np.zeros(thickness.shape,dtype=np.int)
    thickness_map[thickness.data >= 2] = 1
    thickness_map[thickness.data < 2] = 2
    thickness_map[thickness.mask] = 0
    fig = plt.figure(figsize=(3,3),dpi=300)
    ax = fig.add_subplot(111)
    ax.imshow(thickness_map, aspect='auto',cmap=cm, interpolation='none')
    plt.savefig(filename)
    plt.close(fig)

if __name__=='__main__':
    logging.basicConfig(level=logging.DEBUG)
    controls = load_control_data(path = 'Data/Sample/',
                                 laterality='OD')

    p=re.compile('^P(\d{7}).*(\d{1,2}-\d{1,2}-\d{4}).*(OD|OS)_(sn\d+)')
    p2=re.compile('^(.+)_Surfaces_Iowa.xml')

    surface_files = []
    center_files = []
    for dirname, _, filenames in os.walk('Data/Patient/'):
        for name in fnmatch.filter(filenames,'*Surfaces_Iowa.xml'):
            surface_files.append((dirname,name))
        for name in fnmatch.filter(filenames,'*GridCenter_Iowa.xml'):
            center_files.append((dirname,name))

    for surface_file in surface_files:
        basename = re.match(p2,surface_file[1]).group(1)
        try:
            i = [i for i,v in enumerate(center_files) if re.search(basename,v[1])][0]
        except IndexError:
            i = None

        if i is not None:
            patient = Oct.OctLayers(filename = os.path.join(surface_file[0], surface_file[1]),
                                    center_filename = os.path.join(center_files[i][0],center_files[i][1]))
            patient.centerData()
        else:
            logger.warning('GridCenter file not found for surface file:{}'.format(surface_file[1]))
            patient = OctLayers(filename = os.path.join(surface_file[0], surface_file[1]))
        patient.setOrient('OD')

        file_info = re.match(p,surface_file[1])
        layer_set = [(0,10,'Total'),
                     (0,1,'RNFL'),
                     (0,3,'RNFL+GCL'),
                     (0,4,'Inner'),
                     (5,9,'Outer')]

        #open a text file for writing
        tex_basename = '{}-{}-{}-{}'.format(file_info.group(1),
                                            file_info.group(2),
                                            file_info.group(3),
                                            file_info.group(4))
        #create a folder to hold the figures
        tex_figure_folder = os.path.join('Output/',tex_basename)
        if not os.path.exists(tex_figure_folder ):
            os.mkdir(tex_figure_folder)

        #open the tex file for writing
        tex_filename = os.path.join('Output/',tex_basename + '.tex')
        tex_file = open(tex_filename, 'w')

        tex_file.write('\\documentclass{article}\n')
        tex_file.write('\\usepackage{graphicx}\n')
        tex_file.write('\\begin{document}\n')
        tex_file.write('\\title{{{}}}\n'.format(tex_basename))
        tex_file.write('\\maketitle\n')
        for layers in layer_set:
            # generate the figure
            fig_filename = os.path.join(tex_figure_folder,layers[2]+'.png')
            tex_fig_filename = os.path.join(tex_basename,layers[2]+'.png')
            plotLayerZscores(patient, controls, layers[0], layers[1], fig_filename)
            tex_file.write('\\begin{figure}\n')
            tex_file.write('\t\\centering\n')
            tex_file.write('\t\\includegraphics[width=3.0in]{{{}}}\n'.format(tex_fig_filename))
            tex_file.write('\t\\caption{{layer {} to {} - {}}}\n'.format(*layers))
            tex_file.write('\\end{figure}\n')
        tex_file.write('\\end{document}\n')
        tex_file.close()
    <commit_msg>Create pdf analysis output from python - disused, much easier to perform in R<commit_after>import logging
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import os
import re
import datetime
import fnmatch
import dill
import App.OctLayers as Oct
from PIL import Image

def getLayerAsZScore(patient,controls,surface1,surface2):
    layer = patient.getThickness(surface1, surface2)
    layer_mean = controls.getLayerValues(surface1, surface2, 'mean')
    layer_sd = controls.getLayerValues(surface1, surface2, 'stdev')
    layer_zScore = (layer - layer_mean) / layer_sd
    layer_zScore.mask = layer.mask
    return layer_zScore

def getLayerRange(controls,surface1,surface2):
    layer_mean = controls.getLayerValues(surface1, surface2, 'mean')
    layer_sd = controls.getLayerValues(surface1, surface2, 'stdev')

    layer_vals = layer_mean + (4 * layer_sd)
    return (layer_vals.min(), layer_vals.max())

def plotControlLayer(controls,range,surface1,surface2,filename):
    layer = controls.getLayerValues(surface1, surface2, 'mean')
    layer[0,0] = range[0]
    layer[-1,0] = range[1]

    # scale values to be between 0 and 1
    layer = (layer - layer.min()) / layer.max()

    fig = plt.figure(figsize=(3,3),dpi=300)
    ax = fig.add_subplot(111)
    fig.subplots_adjust(top=1, bottom=0, left=0, right=1)
    ax.imshow(layer,
              aspect='auto',
              cmap=plt.get_cmap('jet'),
              interpolation='none')
    plt.savefig(filename,dpi=300)
    plt.close(fig)

def load_control_data(path, laterality, nmax=None):
    #pkl_filename = 'Data/controls_obj.pkl'

    #if os.path.isfile(pkl_filename):
    #    with open(pkl_filename, 'rb') as input:
    #        controls = dill.load(input)
    #else:
    controls = Oct.OctCollection(folder = path,
                                 laterality = laterality,
                                 nmax=nmax)
    # extract the patient id from the filename and append to the OCTLayer object
    p=re.compile('.*([0-9]{3})_[OD|OS]')
    for data in controls.data:
        data.serialNo = re.match(p, data.filename).group(1)
    #    with open(pkl_filename, 'wb') as output:
    #        dill.dump(controls, output)

    return controls

def plotLayer(patient, controls, surface1, surface2, filename):
    layerRange = getLayerRange(controls, surface1, surface2)
    layer = patient.getThickness(surface1, surface2)

    # going to force two pixels to range max and min
    layer[0,0] = layerRange[0]
    layer[-1,0] = layerRange[1]
    layerRange = (layer.min(), layer.max())
    # scale values to be between 0 and 1
    #layer = (layer - layer.min()) / layer.max()

    fig = plt.figure(figsize=(3,3),dpi=300)
    ax = fig.add_subplot(111)
    fig.subplots_adjust(top=1, bottom=0, left=0, right=1)
    im = ax.imshow(layer,
                   aspect='auto',
                   cmap=plt.get_cmap('jet'),
                   interpolation='none')
    fig.colorbar(im)
    plt.savefig(filename,dpi=300)
    plt.close(fig)
    return layerRange

def plotScale(range, filename):
    gradient = np.linspace(0, 1, 255)
    gradient = np.vstack((gradient,gradient)).transpose()
    #gradient = np.flipud(gradient)

    scale = range[1] - range[0]
    gradient = gradient * scale
    gradient = gradient + range[0]

    fig = plt.figure(figsize=(0.75,3),dpi=300)
    ax = fig.add_subplot(111)
    ax.axes.get_xaxis().set_visible(False)
    fig.subplots_adjust(top=1, bottom=0, left=0.5, right=1)
    im = ax.imshow(gradient,
                   aspect='auto',
                   cmap=plt.get_cmap('jet'),
                   interpolation='none')
    fig.colorbar(im)
    plt.savefig(filename,dpi=300)
    plt.close(fig)

def plotLayerZscores(patient,controls,surface1,surface2,filename):
    colors = [(0,0,0),(1,0,0),(0,1,0)]
    cm = mpl.colors.ListedColormap(colors)
    thickness = getLayerAsZScore(patient, controls, surface1, surface2)
    thickness_map = np.zeros(thickness.shape,dtype=np.int)
    thickness_map[thickness.data >= 2] = 1
    thickness_map[thickness.data < 2] = 2
    thickness_map[thickness.mask] = 0
    thickness_map[0,0] = 0
    thickness_map[0,1] = 1
    thickness_map[0,2] = 2
    fig = plt.figure(figsize=(3,3),dpi=300)
    ax = fig.add_subplot(111)
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    fig.subplots_adjust(top=1, bottom=0, left=00, right=1)
    ax.imshow(thickness_map, aspect='auto',cmap=cm, interpolation='none')
    plt.savefig(filename,dpi=300)
    plt.close(fig)

def assemble_images(basepath):
    im_scale = Image.open(basepath + '_scale.png')
    im_layer = Image.open(basepath + '_layer.png')
    im_inset = Image.open(basepath + '_inset.png')
    im_control = Image.open(basepath + '_control.png')

    im_inset.thumbnail((150,150), Image.ANTIALIAS)
    im_control.thumbnail((150,150), Image.ANTIALIAS)

    newWidth = im_scale.size[0] + im_layer.size[0]
    newWidth = im_layer.size[0]
    result = Image.new("RGB", (newWidth,im_scale.size[1]))
    #result.paste(im_scale,(0,0,im_scale.size[0],im_scale.size[1]))
    result.paste(im_layer,(0,0,im_layer.size[0],im_layer.size[1]))
    result.paste(im_inset, (0, 0, im_inset.size[0], im_inset.size[1]))
    result.paste(im_control, (0,
                              result.size[1] - im_control.size[1],
                              im_control.size[0],
                              result.size[1]))
    result.save(basepath + '.png')

if __name__=='__main__':
    logging.basicConfig(level=logging.DEBUG)

    # define the layers of interest
    #    [topSurface,bottomSurface,Name,applyMask]
    #    surfaces are numbered from 0 (not 1 as in segmentation software)
    layer_set = [(0,10,'Total',False),
                 (0,1,'RNFL',True),
                 (1,2,'GCL',True),
                 (2,3,'IPL',True),
                 (1,3,'GCL + IPL',True),
                 (1,4,'GCL + IPL + INL',True),
                 (0,3,'RNFL + GCL + IPL',True),
                 (0,4,'RNFL + GCL + IPL + INL',True),
                 (5,8,'Outer',True),
                 (6,8,'PhR',True),
                 (3,4,'INL',True),
                 (4,5,'OPL',True),
                 (5,6,'ONL',True),
                 (1,5,'Inner Complex',True),
                 (5,8,'Outer Complex',True),
                 (4,8,'Outer Complex + OPL',True),
                 (1,10,'Total GCL-Bruchs',True),
                 (5,10,'Outer Retina',True)]

    # load the control data
    controls = load_control_data(path = 'Data/Control/',
                                 laterality='OD',
                                 nmax=None)

    data_etdrs = []
    for layer in layer_set:
        tmp_etdrs = controls.getEtdrs(layer[0], layer[1], True)
        tmp_etdrs['layer'] = layer[2]
        tmp_etdrs['group'] = 0  # controls
        tmp_etdrs['id'] = 0
        data_etdrs.append(tmp_etdrs)
    data_etdrs = pd.concat(data_etdrs)
    data_etdrs = pd.melt(data_etdrs, id_vars=['layer','group','id'])

    p=re.compile('^P(\d{7}).*(\d{1,2}-\d{1,2}-\d{4}).*(OD|OS)_(sn\d+)')
    p2=re.compile('^(.+)_Surfaces_Iowa.xml')

    surface_files = []
    center_files = []
    # find the files containing the patient data
    for dirname, _, filenames in os.walk('Data/Patient/'):
        for name in fnmatch.filter(filenames,'*Surfaces_Iowa.xml'):
            m = re.match(p,name)
            # convert date format
            dot = datetime.datetime.strptime(m.group(2),'%m-%d-%Y').strftime('%Y-%m-%d')
            surface_files.append({'dirname':dirname,
                                  'filename':name,
                                  'subject':m.group(1),
                                  'dot':dot,
                                  'eye':m.group(3),
                                  'serial':m.group(4)})
        # check if a GridCenter file has been created
        for name in fnmatch.filter(filenames,'*GridCenter_Iowa.xml'):
            center_files.append((dirname,name))

    # sort by subject and test date
    surface_files = sorted(surface_files, key=lambda k: k['subject'] + k['dot'] + k['eye'])

    last_subject = None
    # loop through the patient data
    patient_data = []
    for surface_file in surface_files:
        logging.debug('Serial:{}'.format(surface_file['serial']))

        if not last_subject == surface_file['subject']:
            last_eye = None
            last_subject = surface_file['subject']
            # Chapter for each subject
            # tex_file.write('\\chapter{{Subject: {}}}\n'.format(surface_file['subject']))

        basename = re.match(p2,surface_file['filename']).group(1)
        try:
            i = [i for i,v in enumerate(center_files) if re.search(basename,v[1])][0]
        except IndexError:
            i = None
        # load the patient data and apply the center file if it exists
        if i is not None:
            patient = Oct.OctLayers(filename = os.path.join(surface_file['dirname'], surface_file['filename']),
                                    center_filename = os.path.join(center_files[i][0],center_files[i][1]))
            patient.centerData()
        else:
            logger.warning('GridCenter file not found for surface file:{}'.format(surface_file['filename']))
            patient = Oct.OctLayers(filename = os.path.join(surface_file['dirname'], surface_file['filename']))
        patient.serialNo = surface_file['serial']

        # generate the etdrs data from the patient
        # first we need to set the patient orientation to OD
        orig_orient = surface_file['eye']
        patient.setOrient('OD')
        patient_etdrs = []
        for layer in layer_set:
            tmp_etdrs = patient.getEtdrsThickness(layer[0],layer[1],layer[3])
            tmp_etdrs['layer'] = layer[2]
            tmp_etdrs['group'] = 1 # patients
            tmp_etdrs['id'] = patient.serialNo
            patient_etdrs.append(pd.DataFrame([tmp_etdrs])) #convert from a series to a df with one row
        patient_etdrs = pd.concat(patient_etdrs)
        patient_etdrs = pd.melt(patient_etdrs, id_vars=['layer','group','id'])
        data_etdrs = pd.concat([data_etdrs,patient_etdrs])
        data_etdrs['test']=0
        # reset the orientation
        patient.setOrient(surface_file['eye'])
        # controls.setOrient(surface_file['eye'])
        patient_data.append({'data':patient,
                             'file':surface_file})

    # sort the data into test1, test2 pairs
    testPairs = []
    subjects = set([file['subject']+file['eye'] for file in surface_files])
    for subjecteye in subjects:
        tests = [test for test in surface_files if test['subject']+test['eye'] == subjecteye]
        if len(tests) > 2:
            logger.warning('More than two tests identified for subject:'.format(subjecteye))
        tests = sorted(tests, key=lambda k: k['dot'])
        test1 = tests[0]
        test2 = tests[-1]
        testPairs.append((test1,test2))
        data_etdrs.loc[data_etdrs['id']==test1['serial'],'test']=1
        data_etdrs.loc[data_etdrs['id']==test2['serial'],'test']=2
        data_etdrs.loc[data_etdrs['id']==test1['serial'],'subject']=test1['subject']
        data_etdrs.loc[data_etdrs['id']==test2['serial'],'subject']=test2['subject']

    # save the etdrs data for processing in R
    data_etdrs.to_csv('Data/etdrs_data.csv')

    # open a tex file for writing
    tex_filename = os.path.join('Output/','analysis' + '.tex')
    tex_file = open(tex_filename, 'w')
    # write the document header
    tex_file.write('\\documentclass{report}\n')
    tex_file.write('\\usepackage{graphicx}\n')
    tex_file.write('\\usepackage[space]{grffile}\n') # fix for graphics files with spaces
    tex_file.write('\\usepackage{titlesec}\n')
    tex_file.write('\\usepackage{pdflscape}\n')
    #tex_file.write('\\usepackage[landscape]{geometry}\n')
    tex_file.write('\\newcommand{\\sectionbreak}{\\clearpage}\n')
    tex_file.write('\\newcommand{\\subsectionbreak}{\\clearpage}\n')
    tex_file.write('\\title{{{}}}\n'.format('OCT Thickness Analysis'))
    tex_file.write('\\author{Tom Wright}\n')
    tex_file.write('\\date{{{}}}\n'.format(datetime.datetime.now().strftime("%Y-%m-%d")))

    tex_file.write('\\begin{document}\n')
    tex_file.write('\\maketitle\n')

    for tests in testPairs:
        data_1 = [patient for patient in patient_data if patient['data'].serialNo == tests[0]['serial']]
        data_2 = [patient for patient in patient_data if patient['data'].serialNo == tests[1]['serial']]
        p=(data_1[0],data_2[0])

        controls.setOrient(p[0]['file']['eye'])

        tex_basename = '{}-{}'.format(p[0]['file']['subject'],
                                      p[0]['file']['eye'])

        #tex_file.write('\t\\section{{Test date:{}}}\n'.format(surface_file['dot']))
        # subsection for each test
        tex_file.write('\t\t\\section{{Patient:{}, Eye:{}}}\n'.format(p[0]['file']['subject'],
                                                                      p[0]['file']['eye']))
        #create a folder to hold the figures
        tex_figure_folder = os.path.join('Output/',tex_basename)
        tex_file.write('\t\t\t\\graphicspath{{{{{}/}}}}\n'.format(tex_basename))

        if not os.path.exists(tex_figure_folder ):
            os.mkdir(tex_figure_folder)

        # loop through the layers of interest
        for i,layers in enumerate(layer_set):
            for test in [0,1]:
                # generate the figure
                fig_basename = os.path.join(tex_figure_folder,'test' + '_' + str(test) + '_' + layers[2])
                fig_inset_filename = fig_basename + '_inset.png'
                fig_layer_filename = fig_basename + '_layer.png'
                fig_scale_filename = fig_basename + '_scale.png'
                fig_control_filename = fig_basename + '_control.png'
                tex_fig_filename = os.path.join(tex_basename,layers[2]+'.png')
                plotLayerZscores(p[test]['data'], controls, layers[0], layers[1], fig_inset_filename)
                r = plotLayer(p[test]['data'], controls, layers[0], layers[1], fig_layer_filename)
                plotScale(r, fig_scale_filename)
                plotControlLayer(controls, r, layers[0], layers[1], fig_control_filename)

                assemble_images(fig_basename)

            tex_file.write('\t\t\t\t\\begin{figure}\n')
            tex_file.write('\t\t\t\t\t\\centering\n')
            #if '+' in layers[2]:
                #tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{"test_0_{}".png}}\n'.format(layers[2]).replace(' ','\space '))
                #tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{"test_1_{}".png}}\n'.format(layers[2]).replace(' ','\space '))
            #else:
                #tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{test_0_{}.png}}\n'.format(layers[2]))
                #tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{test_1_{}.png}}\n'.format(layers[2]))
            tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{test_0_{}.png}}\n'.format(layers[2]).replace(' ','\space '))
            tex_file.write('\t\t\t\t\t\\includegraphics[width=3.5in]{{test_1_{}.png}}\n'.format(layers[2]).replace(' ','\space '))
            tex_file.write('\t\t\t\t\t\\caption{{layer {} to {}}}\n'.format(layers[0],layers[1]))
            tex_file.write('\t\t\t\t\t\t{}\n'.format(layers[2]).replace('+','$+$'))
            #tex_file.write('\t\t\t\t\t\\hspace{0.5cm}\n')
            tex_file.write('\t\t\t\t\\end{figure}\n')
            tex_file.write('\t\t\t\\clearpage\n')

    # finish and close the tex doc
    tex_file.write('\\end{document}\n')
    tex_file.close()
    <|endoftext|>


<commit_before>from typing import Optional

import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from lexos.models.base_model import BaseModel
from lexos.helpers.error_messages import NON_NEGATIVE_INDEX_MESSAGE
from lexos.models.matrix_model import MatrixModel
from lexos.receivers.matrix_receiver import IdTempLabelMap
from lexos.receivers.similarity_receiver import SimilarityOption, \
    SimilarityReceiver

class SimilarityModel(BaseModel):
    def __init__(self, test_dtm: Optional[pd.DataFrame] = None,
                 test_option: Optional[SimilarityOption] = None,
                 test_id_temp_label_map: Optional[IdTempLabelMap] = None):
        """This is the class to generate similarity.

        :param test_dtm: (fake parameter)
                         the doc term matrix used for testing.
        :param test_option: (fake parameter)
                            the similarity option used for testing.
        :param test_id_temp_label_map: (fake parameter)
                                       the id temp label map used for testing.
        """
        super().__init__()
        self._test_dtm = test_dtm
        self._test_option = test_option
        self._test_id_temp_label_map = test_id_temp_label_map

    @property
    def _doc_term_matrix(self) -> pd.DataFrame:
        """:return: the document term matrix."""
        return self._test_dtm if self._test_dtm is not None \
            else MatrixModel().get_matrix()

    @property
    def _id_temp_label_map(self) -> IdTempLabelMap:
        """:return: a map takes an id to temp labels."""
        return self._test_id_temp_label_map \
            if self._test_id_temp_label_map is not None \
            else MatrixModel().get_temp_label_id_map()

    @property
    def _similarity_option(self) -> SimilarityOption:
        """:return: the similarity option."""
        return self._test_option if self._test_option is not None \
            else SimilarityReceiver().options_from_front_end()

    def _similarity_maker(self) -> pd.DataFrame:
        """this function generate the result of cos-similarity between files

        :return: docs_score: a parallel list with `docs_name`, is an
                             array of the cos-similarity distance
        :return: docs_name: a parallel list with `docs_score`, is an
                             array of the name (temp labels)
        """
        # precondition
        assert self._similarity_option.comp_file_id >= 0, \
            NON_NEGATIVE_INDEX_MESSAGE

        # get labels
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # get cosine_similarity
        dist = 1 - cosine_similarity(self._doc_term_matrix.values)

        # get an array of file index in file manager files
        other_file_indexes = np.where(self._doc_term_matrix.index !=
                                      self._similarity_option.comp_file_id)[0]
        select_file_indexes = np.where(self._doc_term_matrix.index ==
                                       self._similarity_option.comp_file_id)[0]

        # construct an array of scores
        docs_score_array = np.asarray(
            [dist[file_index, select_file_indexes]
             for file_index in other_file_indexes])

        # construct an array of names
        docs_name_array = np.asarray([labels[i]
                                     for i in list(other_file_indexes)])

        # sort the score array
        sorted_score_array = np.sort(docs_score_array)

        # round the score array to 4 decimals
        final_score_array = np.round(sorted_score_array, decimals=4)

        # sort the name array in terms of the score array
        sorted_score_array_index = docs_score_array.argsort()
        final_name_array = docs_name_array[sorted_score_array_index]

        # pack the scores and names in data_frame
        score_name_data_frame = pd.DataFrame(final_score_array,
                                             index=final_name_array,
                                             columns=["Cosine similarity"])
        return score_name_data_frame

    def get_similarity_score(self) -> pd.DataFrame:

        return self._similarity_maker()
<commit_msg>Testing<commit_after>from typing import Optional

import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from lexos.models.base_model import BaseModel
from lexos.helpers.error_messages import NON_NEGATIVE_INDEX_MESSAGE
from lexos.models.matrix_model import MatrixModel
from lexos.receivers.matrix_receiver import IdTempLabelMap
from lexos.receivers.similarity_receiver import SimilarityOption, \
    SimilarityReceiver

class SimilarityModel(BaseModel):
    def __init__(self, test_dtm: Optional[pd.DataFrame] = None,
                 test_option: Optional[SimilarityOption] = None,
                 test_id_temp_label_map: Optional[IdTempLabelMap] = None):
        """This is the class to generate similarity.

        :param test_dtm: (fake parameter)
                         the doc term matrix used for testing.
        :param test_option: (fake parameter)
                            the similarity option used for testing.
        :param test_id_temp_label_map: (fake parameter)
                                       the id temp label map used for testing.
        """
        super().__init__()
        self._test_dtm = test_dtm
        self._test_option = test_option
        self._test_id_temp_label_map = test_id_temp_label_map

    @property
    def _doc_term_matrix(self) -> pd.DataFrame:
        """:return: the document term matrix."""
        return self._test_dtm if self._test_dtm is not None \
            else MatrixModel().get_matrix()

    @property
    def _id_temp_label_map(self) -> IdTempLabelMap:
        """:return: a map takes an id to temp labels."""
        return self._test_id_temp_label_map \
            if self._test_id_temp_label_map is not None \
            else MatrixModel().get_temp_label_id_map()

    @property
    def _similarity_option(self) -> SimilarityOption:
        """:return: the similarity option."""
        return self._test_option if self._test_option is not None \
            else SimilarityReceiver().options_from_front_end()

    def _similarity_maker(self) -> pd.DataFrame:
        """this function generate the result of cos-similarity between files

        :return: docs_score: a parallel list with `docs_name`, is an
                             array of the cos-similarity distance
        :return: docs_name: a parallel list with `docs_score`, is an
                             array of the name (temp labels)
        """
        # precondition
        assert self._similarity_option.comp_file_id >= 0, \
            NON_NEGATIVE_INDEX_MESSAGE

        # get labels
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # get cosine_similarity
        dist = 1 - cosine_similarity(self._doc_term_matrix.values)

        # get an array of file index in file manager files
        other_file_indexes = np.where(self._doc_term_matrix.index !=
                                      self._similarity_option.comp_file_id)[0]
        select_file_indexes = np.where(self._doc_term_matrix.index ==
                                       self._similarity_option.comp_file_id)[0]

        # construct an array of scores
        Test = [dist[file_index, select_file_indexes]
             for file_index in other_file_indexes]
        docs_score_array = np.asarray(
            [dist[file_index, select_file_indexes]
             for file_index in other_file_indexes])

        # construct an array of names
        compared_file_label = np.asarray(
            [self._id_temp_label_map[file_id]
             for file_id in self._doc_term_matrix.index.values
             if file_id != self._similarity_option.comp_file_id])
        docs_name_array = np.asarray([labels[i]
                                     for i in list(other_file_indexes)])

        # sort the score array
        sorted_score_array = np.sort(docs_score_array)

        # round the score array to 4 decimals
        final_score_array = np.round(sorted_score_array, decimals=4)

        # sort the name array in terms of the score array
        sorted_score_array_index = docs_score_array.argsort()
        final_name_array = docs_name_array[sorted_score_array_index]
        sorted_compared_file_label = compared_file_label[sorted_score_array_index]

        # pack the scores and names in data_frame
        score_name_data_frame = pd.DataFrame(final_score_array,
                                             index=final_name_array,
                                             columns=["Cosine similarity"])
        return score_name_data_frame

    def get_similarity_score(self) -> pd.DataFrame:

        return self._similarity_maker()
<|endoftext|>