<commit_before># This file is part of LiSE, a framework for life simulation games.
# Copyright (C) 2013-2014 Zachary Spector, ZacharySpector@gmail.com
"""Widget to display the contents of a :class:`kivy.atlas.Atlas` in
one :class:`kivy.uix.togglebutton.ToggleButton` apiece, arranged in a
:class:`kivy.uix.stacklayout.StackLayout`. The user selects graphics
from the :class:`Pallet`, and the :class:`Pallet` updates its
``selection`` list to show what the user selected."""
from kivy.properties import (
    DictProperty,
    NumericProperty,
    ObjectProperty,
    OptionProperty,
    ListProperty,
    ReferenceListProperty,
    StringProperty
)
from kivy.resources import resource_find, resource_paths
from kivy.atlas import Atlas
from kivy.lang import Builder
from kivy.logger import Logger
from kivy.uix.togglebutton import ToggleButton
from kivy.uix.stacklayout import StackLayout

class SwatchButton(ToggleButton):
    """Toggle button containing a texture and its name, which, when
    toggled, will report the fact to the :class:`Pallet` it's in.

    """
    name = StringProperty()
    tex = ObjectProperty()

    def on_state(self, *args):
        if self.state == 'down':
            assert(self not in self.parent.selection)
            if self.parent.selection_mode == 'single':
                self.parent.selection = [self]
            else:
                self.parent.selection.append(self)
        else:
            if self in self.parent.selection:
                self.parent.selection.remove(self)

kv = """
<SwatchButton>:
    Image:
        id: theimg
        center: root.center
        texture: root.tex
        size: root.tex.size if root.tex else (1, 1)
        size_hint: (None, None)
        pos_hint: {'x': None, 'y': None}
    Label:
        text: root.name
        size: self.texture_size
        pos_hint: {'x': None, 'y': None}
        x: root.x
        top: theimg.y
"""
Builder.load_string(kv)

class Pallet(StackLayout):
    atlas = ObjectProperty()
    filename = StringProperty()
    swatches = DictProperty({})
    swatch_width = NumericProperty(100)
    swatch_height = NumericProperty(75)
    swatch_size = ReferenceListProperty(swatch_width, swatch_height)
    selection = ListProperty([])
    selection_mode = OptionProperty('single', options=['single', 'multiple'])

    def on_filename(self, *args):
        if not self.filename:
            return
        Logger.debug('Pallet: searching for atlas in resource path {}'.format(resource_paths))
        self.atlas = Atlas(resource_find(self.filename))

    def on_atlas(self, *args):
        if self.atlas is None:
            return
        self.upd_textures()
        self.atlas.bind(textures=self.upd_textures)

    def upd_textures(self, *args):
        for name in list(self.swatches.keys()):
            if name not in self.atlas.textures:
                self.remove_widget(self.swatches[name])
                del self.swatches[name]
        for (name, tex) in self.atlas.textures.items():
            if name in self.swatches and self.swatches[name] != tex:
                self.remove_widget(self.swatches[name])
            if name not in self.swatches or self.swatches[name] != tex:
                self.swatches[name] = SwatchButton(
                    name=name,
                    tex=tex,
                    size_hint=(None, None),
                    size=self.swatch_size
                )
                self.add_widget(self.swatches[name])

kv = """
<Pallet>:
    orientation: 'lr-tb'
    padding_y: 100
    size_hint: (None, None)
"""
Builder.load_string(kv)
<commit_msg>make SwatchButton look nicer<commit_after># This file is part of LiSE, a framework for life simulation games.
# Copyright (C) 2013-2014 Zachary Spector, ZacharySpector@gmail.com
"""Widget to display the contents of a :class:`kivy.atlas.Atlas` in
one :class:`kivy.uix.togglebutton.ToggleButton` apiece, arranged in a
:class:`kivy.uix.stacklayout.StackLayout`. The user selects graphics
from the :class:`Pallet`, and the :class:`Pallet` updates its
``selection`` list to show what the user selected."""
from kivy.properties import (
    DictProperty,
    NumericProperty,
    ObjectProperty,
    OptionProperty,
    ListProperty,
    ReferenceListProperty,
    StringProperty
)
from kivy.resources import resource_find, resource_paths
from kivy.atlas import Atlas
from kivy.lang import Builder
from kivy.logger import Logger
from kivy.uix.togglebutton import ToggleButton
from kivy.uix.stacklayout import StackLayout

class SwatchButton(ToggleButton):
    """Toggle button containing a texture and its name, which, when
    toggled, will report the fact to the :class:`Pallet` it's in.

    """
    name = StringProperty()
    tex = ObjectProperty()

    def on_state(self, *args):
        if self.state == 'down':
            assert(self not in self.parent.selection)
            if self.parent.selection_mode == 'single':
                for wid in self.parent.selection:
                    if wid is not self:
                        wid.state = 'normal'
                self.parent.selection = [self]
            else:
                self.parent.selection.append(self)
        else:
            if self in self.parent.selection:
                self.parent.selection.remove(self)

kv = """
<SwatchButton>:
    Image:
        id: theimg
        center: root.center
        texture: root.tex
        size: root.tex.size if root.tex else (1, 1)
        size_hint: (None, None)
        pos_hint: {'x': None, 'y': None}
    Label:
        text: root.name
        size: self.texture_size
        pos_hint: {'x': None, 'y': None}
        x: root.x + 5
        y: theimg.y - self.height
"""
Builder.load_string(kv)

class Pallet(StackLayout):
    atlas = ObjectProperty()
    filename = StringProperty()
    swatches = DictProperty({})
    swatch_width = NumericProperty(100)
    swatch_height = NumericProperty(75)
    swatch_size = ReferenceListProperty(swatch_width, swatch_height)
    selection = ListProperty([])
    selection_mode = OptionProperty('single', options=['single', 'multiple'])

    def on_filename(self, *args):
        if not self.filename:
            return
        self.atlas = Atlas(resource_find(self.filename))

    def on_atlas(self, *args):
        if self.atlas is None:
            return
        self.upd_textures()
        self.atlas.bind(textures=self.upd_textures)

    def upd_textures(self, *args):
        for name in list(self.swatches.keys()):
            if name not in self.atlas.textures:
                self.remove_widget(self.swatches[name])
                del self.swatches[name]
        for (name, tex) in self.atlas.textures.items():
            if name in self.swatches and self.swatches[name] != tex:
                self.remove_widget(self.swatches[name])
            if name not in self.swatches or self.swatches[name] != tex:
                self.swatches[name] = SwatchButton(
                    name=name,
                    tex=tex,
                    size_hint=(None, None),
                    size=self.swatch_size
                )
                self.add_widget(self.swatches[name])

kv = """
<Pallet>:
    orientation: 'lr-tb'
    padding_y: 100
    size_hint: (None, None)
"""
Builder.load_string(kv)
<|endoftext|>


<commit_before>"""
Rotor class for Enigma Machine

"""

class Rotor:
    """Basic Rotor class for Enigma Machine"""
    rotorNum = 0

    def __init__(self, configuration):
        self.config = configuration
        self.POSITION = 0
        Rotor.rotorNum += 1

    def rotate(self):
        #Increments the current Rotor position
        self.POSITION += 1

    def setStart(self, position):
        #Takes starting position from 0 to n on the Rotor
        self.POSITION = position

<commit_msg>updated rotate() to accomodate for rotors that cant rotate<commit_after>"""
Rotor class for Enigma Machine

"""

class Rotor:
    """Basic Rotor class for Enigma Machine"""
    rotorNum = 0

    def __init__(self, configuration):
        self.config = configuration
        self.POSITION = 0
        Rotor.rotorNum += 1

    def rotate(self):
        #Increments the current Rotor position
        if len(self.config) > 1:
            self.POSITION += 1
        else:
            print "the rotor is not large enough to rotate"
            pass

    def setStart(self, position):
        #Takes starting position from 0 to n on the Rotor
        self.POSITION = position

<|endoftext|>


<commit_before>import asyncio
import concurrent.futures
import io
import sys
from typing import Iterable, List, Optional, Tuple

from asgiref.typing import (
    ASGIReceiveCallable,
    ASGIReceiveEvent,
    ASGISendCallable,
    ASGISendEvent,
    HTTPRequestEvent,
    HTTPResponseBodyEvent,
    HTTPResponseStartEvent,
    HTTPScope,
)

from uvicorn._types import Environ, ExcInfo, StartResponse, WSGIApp

def build_environ(scope: HTTPScope, message: ASGIReceiveEvent, body: bytes) -> Environ:
    """
    Builds a scope and request message into a WSGI environ object.
    """
    environ = {
        "REQUEST_METHOD": scope["method"],
        "SCRIPT_NAME": "",
        "PATH_INFO": scope["path"].encode("utf8").decode("latin1"),
        "QUERY_STRING": scope["query_string"].decode("ascii"),
        "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"],
        "wsgi.version": (1, 0),
        "wsgi.url_scheme": scope.get("scheme", "http"),
        "wsgi.input": io.BytesIO(body),
        "wsgi.errors": sys.stdout,
        "wsgi.multithread": True,
        "wsgi.multiprocess": True,
        "wsgi.run_once": False,
    }

    # Get server name and port - required in WSGI, not in ASGI
    server = scope.get("server")
    if server is None:
        server = ("localhost", 80)
    environ["SERVER_NAME"] = server[0]
    environ["SERVER_PORT"] = server[1]

    # Get client IP address
    client = scope.get("client")
    if client is not None:
        environ["REMOTE_ADDR"] = client[0]

    # Go through headers and make them into environ entries
    for name, value in scope.get("headers", []):
        name_str: str = name.decode("latin1")
        if name_str == "content-length":
            corrected_name = "CONTENT_LENGTH"
        elif name_str == "content-type":
            corrected_name = "CONTENT_TYPE"
        else:
            corrected_name = "HTTP_%s" % name_str.upper().replace("-", "_")
        # HTTPbis say only ASCII chars are allowed in headers, but we latin1
        # just in case
        value_str: str = value.decode("latin1")
        if corrected_name in environ:
            corrected_name_environ = environ[corrected_name]
            assert isinstance(corrected_name_environ, str)
            value_str = corrected_name_environ + "," + value_str
        environ[corrected_name] = value_str
    return environ

class WSGIMiddleware:
    def __init__(self, app: WSGIApp, workers: int = 10):
        self.app = app
        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)

    async def __call__(
        self, scope: HTTPScope, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        assert scope["type"] == "http"
        instance = WSGIResponder(self.app, self.executor, scope)
        await instance(receive, send)

class WSGIResponder:
    def __init__(
        self,
        app: WSGIApp,
        executor: concurrent.futures.ThreadPoolExecutor,
        scope: HTTPScope,
    ):
        self.app = app
        self.executor = executor
        self.scope = scope
        self.status = None
        self.response_headers = None
        self.send_event = asyncio.Event()
        self.send_queue: List[Optional[ASGISendEvent]] = []
        self.loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()
        self.response_started = False
        self.exc_info: Optional[ExcInfo] = None

    async def __call__(
        self, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        message: HTTPRequestEvent = await receive()  # type: ignore[assignment]
        body = message.get("body", b"")
        more_body = message.get("more_body", False)
        while more_body:
            body_message: HTTPRequestEvent = await receive()  # type: ignore[assignment]
            body += body_message.get("body", b"")
            more_body = body_message.get("more_body", False)
        environ = build_environ(self.scope, message, body)
        self.loop = asyncio.get_event_loop()
        wsgi = self.loop.run_in_executor(
            self.executor, self.wsgi, environ, self.start_response
        )
        sender = self.loop.create_task(self.sender(send))
        try:
            await asyncio.wait_for(wsgi, None)
        finally:
            self.send_queue.append(None)
            self.send_event.set()
            await asyncio.wait_for(sender, None)
        if self.exc_info is not None:
            raise self.exc_info[0].with_traceback(self.exc_info[1], self.exc_info[2])

    async def sender(self, send: ASGISendCallable) -> None:
        while True:
            if self.send_queue:
                message = self.send_queue.pop(0)
                if message is None:
                    return
                await send(message)
            else:
                await self.send_event.wait()
                self.send_event.clear()

    def start_response(
        self,
        status: str,
        response_headers: Iterable[Tuple[str, str]],
        exc_info: Optional[ExcInfo] = None,
    ) -> None:
        self.exc_info = exc_info
        if not self.response_started:
            self.response_started = True
            status_code_str, _ = status.split(" ", 1)
            status_code = int(status_code_str)
            headers = [
                (name.encode("ascii"), value.encode("ascii"))
                for name, value in response_headers
            ]
            http_response_start_event: HTTPResponseStartEvent = {
                "type": "http.response.start",
                "status": status_code,
                "headers": headers,
            }
            self.send_queue.append(http_response_start_event)
            self.loop.call_soon_threadsafe(self.send_event.set)

    def wsgi(self, environ: Environ, start_response: StartResponse) -> None:
        for chunk in self.app(environ, start_response):  # type: ignore
            response_body: HTTPResponseBodyEvent = {
                "type": "http.response.body",
                "body": chunk,
                "more_body": True,
            }
            self.send_queue.append(response_body)
            self.loop.call_soon_threadsafe(self.send_event.set)

        empty_body: HTTPResponseBodyEvent = {
            "type": "http.response.body",
            "body": b"",
            "more_body": False,
        }
        self.send_queue.append(empty_body)
        self.loop.call_soon_threadsafe(self.send_event.set)
<commit_msg>[middleware/wsgi.py] Replace list with deque for `send_queue` (#1214)<commit_after>import asyncio
import concurrent.futures
import io
import sys
from collections import deque
from typing import Deque, Iterable, Optional, Tuple

from asgiref.typing import (
    ASGIReceiveCallable,
    ASGIReceiveEvent,
    ASGISendCallable,
    ASGISendEvent,
    HTTPRequestEvent,
    HTTPResponseBodyEvent,
    HTTPResponseStartEvent,
    HTTPScope,
)

from uvicorn._types import Environ, ExcInfo, StartResponse, WSGIApp

def build_environ(scope: HTTPScope, message: ASGIReceiveEvent, body: bytes) -> Environ:
    """
    Builds a scope and request message into a WSGI environ object.
    """
    environ = {
        "REQUEST_METHOD": scope["method"],
        "SCRIPT_NAME": "",
        "PATH_INFO": scope["path"].encode("utf8").decode("latin1"),
        "QUERY_STRING": scope["query_string"].decode("ascii"),
        "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"],
        "wsgi.version": (1, 0),
        "wsgi.url_scheme": scope.get("scheme", "http"),
        "wsgi.input": io.BytesIO(body),
        "wsgi.errors": sys.stdout,
        "wsgi.multithread": True,
        "wsgi.multiprocess": True,
        "wsgi.run_once": False,
    }

    # Get server name and port - required in WSGI, not in ASGI
    server = scope.get("server")
    if server is None:
        server = ("localhost", 80)
    environ["SERVER_NAME"] = server[0]
    environ["SERVER_PORT"] = server[1]

    # Get client IP address
    client = scope.get("client")
    if client is not None:
        environ["REMOTE_ADDR"] = client[0]

    # Go through headers and make them into environ entries
    for name, value in scope.get("headers", []):
        name_str: str = name.decode("latin1")
        if name_str == "content-length":
            corrected_name = "CONTENT_LENGTH"
        elif name_str == "content-type":
            corrected_name = "CONTENT_TYPE"
        else:
            corrected_name = "HTTP_%s" % name_str.upper().replace("-", "_")
        # HTTPbis say only ASCII chars are allowed in headers, but we latin1
        # just in case
        value_str: str = value.decode("latin1")
        if corrected_name in environ:
            corrected_name_environ = environ[corrected_name]
            assert isinstance(corrected_name_environ, str)
            value_str = corrected_name_environ + "," + value_str
        environ[corrected_name] = value_str
    return environ

class WSGIMiddleware:
    def __init__(self, app: WSGIApp, workers: int = 10):
        self.app = app
        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)

    async def __call__(
        self, scope: HTTPScope, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        assert scope["type"] == "http"
        instance = WSGIResponder(self.app, self.executor, scope)
        await instance(receive, send)

class WSGIResponder:
    def __init__(
        self,
        app: WSGIApp,
        executor: concurrent.futures.ThreadPoolExecutor,
        scope: HTTPScope,
    ):
        self.app = app
        self.executor = executor
        self.scope = scope
        self.status = None
        self.response_headers = None
        self.send_event = asyncio.Event()
        self.send_queue: Deque[Optional[ASGISendEvent]] = deque()
        self.loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()
        self.response_started = False
        self.exc_info: Optional[ExcInfo] = None

    async def __call__(
        self, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        message: HTTPRequestEvent = await receive()  # type: ignore[assignment]
        body = message.get("body", b"")
        more_body = message.get("more_body", False)
        while more_body:
            body_message: HTTPRequestEvent = await receive()  # type: ignore[assignment]
            body += body_message.get("body", b"")
            more_body = body_message.get("more_body", False)
        environ = build_environ(self.scope, message, body)
        self.loop = asyncio.get_event_loop()
        wsgi = self.loop.run_in_executor(
            self.executor, self.wsgi, environ, self.start_response
        )
        sender = self.loop.create_task(self.sender(send))
        try:
            await asyncio.wait_for(wsgi, None)
        finally:
            self.send_queue.append(None)
            self.send_event.set()
            await asyncio.wait_for(sender, None)
        if self.exc_info is not None:
            raise self.exc_info[0].with_traceback(self.exc_info[1], self.exc_info[2])

    async def sender(self, send: ASGISendCallable) -> None:
        while True:
            if self.send_queue:
                message = self.send_queue.popleft()
                if message is None:
                    return
                await send(message)
            else:
                await self.send_event.wait()
                self.send_event.clear()

    def start_response(
        self,
        status: str,
        response_headers: Iterable[Tuple[str, str]],
        exc_info: Optional[ExcInfo] = None,
    ) -> None:
        self.exc_info = exc_info
        if not self.response_started:
            self.response_started = True
            status_code_str, _ = status.split(" ", 1)
            status_code = int(status_code_str)
            headers = [
                (name.encode("ascii"), value.encode("ascii"))
                for name, value in response_headers
            ]
            http_response_start_event: HTTPResponseStartEvent = {
                "type": "http.response.start",
                "status": status_code,
                "headers": headers,
            }
            self.send_queue.append(http_response_start_event)
            self.loop.call_soon_threadsafe(self.send_event.set)

    def wsgi(self, environ: Environ, start_response: StartResponse) -> None:
        for chunk in self.app(environ, start_response):  # type: ignore
            response_body: HTTPResponseBodyEvent = {
                "type": "http.response.body",
                "body": chunk,
                "more_body": True,
            }
            self.send_queue.append(response_body)
            self.loop.call_soon_threadsafe(self.send_event.set)

        empty_body: HTTPResponseBodyEvent = {
            "type": "http.response.body",
            "body": b"",
            "more_body": False,
        }
        self.send_queue.append(empty_body)
        self.loop.call_soon_threadsafe(self.send_event.set)
<|endoftext|>


<commit_before>from argparse import ArgumentParser

argp = ArgumentParser(description='Plays GIF animation or draws images on to the frame buffer.')
argp.add_argument('paths', nargs='*', type=str, help='Paths to the directory to scan or to the file to read. Empty list makes it fall into color test mode.')
argp.add_argument('-ct', '--color-test', action='store_true', help='Beautiful color testing mode')
argp.add_argument('-nc', '--no-clear', action='store_true', help='Do not clear screen with black on startup and exit.')
argp.add_argument('-nl', '--no-loop', action='store_true', help='Make entire playlist not to loop forever.')
argp.add_argument('-nr', '--no-recurse', action='store_true', help='Set to don\'t recurse scan into directories.')
argp.add_argument('-gd', '--global-delay', type=float, help='This time (seconds) applies to entire static/animated images to show for. Animated files will be infinitely looped while this delay. This value will be applied precedece than the \'-sd\' and \'-ad\', \'-al\'.')
argp.add_argument('-sd', '--static-delay', type=float, default=20, help='This time (seconds) applies to only static images to show for.')
argp.add_argument('-al', '--animate-loop', type=int, help='This time (times) applies to only animated images to show for. Animated files will be looped for this count. If it\'s not applied, GIF files will be played for only once of its loop. It precedes \'animate-delay\'.')
argp.add_argument('-ad', '--animate-delay', type=float, help='This time (seconds) applies to only animated images to show for. Animated files will be infinitely looped while this delay. If it\'s not applied, GIF files will be played for only once of its loop.')
argp.add_argument('-pv', '--preview', action='store_true', help='Freeze on first frame while loading next frames to preview before playing')
argp.add_argument('-fb', type=int, default=0, help='Selects frame buffer driver. /dev/fb[-fb] will be used.')
argp.add_argument('-w', '--win', type=str, help='Virtual window setting. Form should be : x1,y1,x2,y2. x2 and y2 can be \'w\' or \'h\' to replaced with framebuffer width and height')
argp.add_argument('-bpp', type=int, default=None, help='Bit per pixels. Most framebuffer drivers uses 24 for 8 bits on each BGR as default. It tries to update the value and fallback to available driver specified value in case of failure. If nothing\'s provided, it uses current framebuffer setting.\n See framebuffer_depth boot config for details on rpi.')
argp.add_argument('-sf', '--shuffle', action='store_true', help='Shuffle the playlist')
argp.add_argument('-vb', '--verbose', action='store_true', help='Show verbose information')
def hex_int(string):
  try:
    v = int('0x'+string, 16)
    if v <= 0xffffff and v >= 0 :
      return v
    else:
      print(string + " is not in a valid range(0 <= x <= ffffff). Replaced with 0(black).")
  except:
    print(string + " is not a valid hex value(ex. 'FF0000' for red). Replaced with 0(black).")
  return 0
argp.add_argument('-c', '--clear', default=-1, type=hex_int, nargs='?', help='Clear the screen with black by default, or give an RGB code to fill with. ex)FF0000 : pure red, BBBBBB : light grey, FF00FF : fuchsia purple.')
args = argp.parse_args()

if args.win: args.win = tuple(args.win.split(','))
if args is None: args = 0
if __name__ == '__main__':
  # argument test code
  print(args)
  argp.print_help()
elif not args.paths and not args.color_test and clear == -1:
  argp.print_help()
<commit_msg>color fill bug fix<commit_after>from argparse import ArgumentParser

argp = ArgumentParser(description='Plays GIF animation or draws images on to the frame buffer.')
argp.add_argument('paths', nargs='*', type=str, help='Paths to the directory to scan or to the file to read. Empty list makes it fall into color test mode.')
argp.add_argument('-ct', '--color-test', action='store_true', help='Beautiful color testing mode')
argp.add_argument('-nc', '--no-clear', action='store_true', help='Do not clear screen with black on startup and exit.')
argp.add_argument('-nl', '--no-loop', action='store_true', help='Make entire playlist not to loop forever.')
argp.add_argument('-nr', '--no-recurse', action='store_true', help='Set to don\'t recurse scan into directories.')
argp.add_argument('-gd', '--global-delay', type=float, help='This time (seconds) applies to entire static/animated images to show for. Animated files will be infinitely looped while this delay. This value will be applied precedece than the \'-sd\' and \'-ad\', \'-al\'.')
argp.add_argument('-sd', '--static-delay', type=float, default=20, help='This time (seconds) applies to only static images to show for.')
argp.add_argument('-al', '--animate-loop', type=int, help='This time (times) applies to only animated images to show for. Animated files will be looped for this count. If it\'s not applied, GIF files will be played for only once of its loop. It precedes \'animate-delay\'.')
argp.add_argument('-ad', '--animate-delay', type=float, help='This time (seconds) applies to only animated images to show for. Animated files will be infinitely looped while this delay. If it\'s not applied, GIF files will be played for only once of its loop.')
argp.add_argument('-pv', '--preview', action='store_true', help='Freeze on first frame while loading next frames to preview before playing')
argp.add_argument('-fb', type=int, default=0, help='Selects frame buffer driver. /dev/fb[-fb] will be used.')
argp.add_argument('-w', '--win', type=str, help='Virtual window setting. Form should be : x1,y1,x2,y2. x2 and y2 can be \'w\' or \'h\' to replaced with framebuffer width and height')
argp.add_argument('-bpp', type=int, default=None, help='Bit per pixels. Most framebuffer drivers uses 24 for 8 bits on each BGR as default. It tries to update the value and fallback to available driver specified value in case of failure. If nothing\'s provided, it uses current framebuffer setting.\n See framebuffer_depth boot config for details on rpi.')
argp.add_argument('-sf', '--shuffle', action='store_true', help='Shuffle the playlist')
argp.add_argument('-vb', '--verbose', action='store_true', help='Show verbose information')
def hex_int(string):
  try:
    v = int('0x'+string, 16)
    if v <= 0xffffff and v >= 0 :
      return v
    else:
      print(string + " is not in a valid range(0 <= x <= ffffff). Replaced with 0(black).")
  except:
    print(string + " is not a valid hex value(ex. 'FF0000' for red). Replaced with 0(black).")
  return 0
argp.add_argument('-c', '--clear', default=-1, type=hex_int, nargs='?', help='Clear the screen with black by default, or give an RGB code to fill with. ex)FF0000 : pure red, BBBBBB : light grey, FF00FF : fuchsia purple.')
args = argp.parse_args()

if args.win: args.win = tuple(args.win.split(','))
if args is None: args = 0
if __name__ == '__main__':
  # argument test code
  print(args)
  argp.print_help()
elif not args.paths and not args.color_test and args.clear == -1:
  argp.print_help()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" The pump module contains the Pump class, which takes care of moving bytes between the upstream and downstream connections. """

import logging
from traceback import print_exc

import monocle
monocle.init('tornado')

from monocle import _o, Return

from monocle.stack.network import ConnectionLost

from obfsproxy.util import encode
from obfsproxy.framework.circuit import Circuit

class Pump(object):

    """ The Pump class takes care of moving bytes between the upstream and downstream connections. """

    def __init__(
        self,
        downstream,
        upstream,
        transportClass,
        ):
        """ Initializes the downstream and upstream instance variables, instantiates the transportClass, and sets up a circuit. """

        self.downstream = downstream
        self.upstream = upstream

        circuit = Circuit()
        self.transport = transportClass(circuit)
        self.circuit = circuit.invert()

    @_o
    def run(self):
        """ Calls the start event on the transport and initiates pumping between upstream and downstream connections in both directions. """

        self.transport.start()

        self.drain()

        monocle.launch(self.pumpUpstream)
        yield self.pumpDownstream()

    @_o
    def drain(self):
        logging.error('drain')
        yield self.pumpOut(self.circuit.downstream, self.downstream)
        yield self.pumpOut(self.circuit.upstream, self.upstream)

    @_o
    def pumpIn(
        self,
        input,
        output,
        callback,
        ):

        logging.error('pumpIn yielding '+str(input))
	try:
	    data = yield input.read_some()
        except ConnectionLost:
            logging.error('pumpIn: Connection lost')
            yield Return(False)
        except IOError:
            print 'IOError'
	    print_exc()
            yield Return(False)
        except Exception, e:
            print 'Exception'
            print e
            yield Return(False)
        if data:
            logging.error('pumpIn read ' + str(len(data)))
            try:
	        output.write(data)
		logging.error('pumpIn wrote %d' % (len(data)))
                callback()
            except ConnectionLost:
                print 'pumpIn: Connection lost'
                yield Return(False)
            except IOError:
                print 'IOError'
		print_exc()
                yield Return(False)
            except Exception, e:
                print 'Exception'
                print e
                yield Return(False)
	else:
		logging.error('pumpIn no data')

        yield Return(True)

    @_o
    def pumpOut(self, input, output):
        logging.error('pumpOut yield')
	try:
            data = input.read_some()
        except ConnectionLost:
            print 'pumpOut: Connection lost'
            return
        except IOError:
            print 'IOError'
	    print_exc()

            return
        except Exception, e:
            print 'Exception'
            print e
            return
        if data:
            logging.error('pumpOut read ' + str(len(data)))
            try:
                yield output.write(data)
		logging.error('pumpOut wrote %d' % (len(data)))
            except ConnectionLost:
                print 'pumpOut: Connection lost'
                return
            except IOError:
                print 'IOError'
		print_exc()

                return
            except Exception, e:
                print 'Exception'
                print e
                return
	else:
	    logging.error('pumpOut no data')

    @_o
    def pumpUpstream(self):
        pumping=True
        while pumping:
	    logging.error('pump upstream')
            pumping=yield self.pumpIn(self.downstream, self.circuit.downstream,
                              self.transport.receivedDownstream)
            yield self.drain()

    @_o
    def pumpDownstream(self):
        pumping=True
        while pumping:
            logging.error('pump downstream')
            pumping=yield self.pumpIn(self.upstream, self.circuit.upstream,
                              self.transport.receivedUpstream)
            yield self.drain()
<commit_msg>Better exception debugging<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" The pump module contains the Pump class, which takes care of moving bytes between the upstream and downstream connections. """

import logging
from traceback import print_exc

import monocle
monocle.init('tornado')

from monocle import _o, Return

from monocle.stack.network import ConnectionLost

from obfsproxy.util import encode
from obfsproxy.framework.circuit import Circuit

class Pump(object):

    """ The Pump class takes care of moving bytes between the upstream and downstream connections. """

    def __init__(
        self,
        downstream,
        upstream,
        transportClass,
        ):
        """ Initializes the downstream and upstream instance variables, instantiates the transportClass, and sets up a circuit. """

        self.downstream = downstream
        self.upstream = upstream

        circuit = Circuit()
        self.transport = transportClass(circuit)
        self.circuit = circuit.invert()

    @_o
    def run(self):
        """ Calls the start event on the transport and initiates pumping between upstream and downstream connections in both directions. """

        self.transport.start()

        self.drain()

        monocle.launch(self.pumpUpstream)
        yield self.pumpDownstream()

    @_o
    def drain(self):
        logging.error('drain')
        yield self.pumpOut(self.circuit.downstream, self.downstream)
        yield self.pumpOut(self.circuit.upstream, self.upstream)

    @_o
    def pumpIn(
        self,
        input,
        output,
        callback,
        ):

        logging.error('pumpIn yielding '+str(input))
        try:
            data = yield input.read_some()
        except ConnectionLost as e:
            logging.error('pumpIn: Connection lost')
            yield Return(False)
        except IOError:
            print 'IOError'
            print_exc()
            yield Return(False)
        except Exception, e:
            print 'Exception'
            print e
            yield Return(False)

        if data:
            logging.error('pumpIn read ' + str(len(data)))
            try:
                output.write(data)
                logging.error('pumpIn wrote %d' % (len(data)))
                callback()
            except ConnectionLost:
                print 'pumpIn: Connection lost'
                yield Return(False)
            except IOError:
                print 'IOError'
                print_exc()
                yield Return(False)
            except Exception, e:
                print 'Exception'
                print e
                yield Return(False)
        else:
            logging.error('pumpIn no data')

        logging.error('pumpIn returning True')
        yield Return(True)

    @_o
    def pumpOut(self, input, output):
        logging.error('pumpOut yield')
        try:
            data = input.read_some()
        except ConnectionLost:
            print 'pumpOut: Connection lost'
            return
        except IOError:
            print 'IOError'
            print_exc()
            return
        except Exception, e:
            print 'Exception'
            print e
            return

        if data:
            logging.error('pumpOut read ' + str(len(data)))
            try:
                yield output.write(data)
                logging.error('pumpOut wrote %d' % (len(data)))
            except ConnectionLost:
                print 'pumpOut: Connection lost'
                return
            except IOError:
                print 'IOError'
                print_exc()
                return
            except Exception, e:
                print 'Exception'
                print e
                return
        else:
            logging.error('pumpOut no data')

    @_o
    def pumpUpstream(self):
        try:
            pumping=True
            while pumping:
                logging.error('pump upstream')
                pumping=yield self.pumpIn(self.downstream, self.circuit.downstream, self.transport.receivedDownstream)
                yield self.drain()
                logging.error('pumping: '+str(pumping))
        except Exception as e:
            logging.error('Exception in pumpUpstream')
            logging.error(e)

    @_o
    def pumpDownstream(self):
        try:
            pumping=True
            while pumping:
                logging.error('pump downstream')
                pumping=yield self.pumpIn(self.upstream, self.circuit.upstream, self.transport.receivedUpstream)
                yield self.drain()
        except Exception as e:
            logging.error('Exception in pumpDownstream')
            logging.error(e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Decorators and other functions for hooks.

@author: Seva Zhidkov
@contact: zhidkovseva@gmail.com
@license: The MIT license

Copyright (C) 2015
"""
import sys
import _thread as thread
import re

from sheldon.exceptions import catch_module_errors

class Hook:
    """
    Basic class for hooks
    """
    # Every hook has type. It could be
    # 'command' (command hook), 'message' (message hook),
    # 'interval' (interval hook), 'cron' (cron hook),
    # 'web' (web hook)
    type = ''
    func = None

    # If not one module return True after check,
    # bot sort matches by priority and choose first.
    priority = 0

    @catch_module_errors
    def call(self, incoming_message, bot):
        """
        Run hook

        :param incoming_message: Message object
        :param bot: Sheldon object
        :return:
        """
        thread.start_new_thread(self.func, (
            incoming_message, bot
        ))

class MessageHook(Hook):
    def __init__(self, user_function, regexes, case_sensitive=False):
        """
        Create new message hook

        :param user_function: decorated user's function, called
                              when regular expression matching with
                              incoming message
        :param regexes: regular expression for messages
        :param case_sensitive: bool, is hook catching messages
                                     with case sensitive
        """
        self.type = 'message'
        self.priority = 2

        self.func = user_function

        if type(regexes) == str:
            regexes = [regexes]
        self.regexes = regexes

        self.case_sensitive = case_sensitive

    def check(self, incoming_message):
        """
        Check, is this message catching for this hook

        :param incoming_message: IncomingMessage object
        :return: True or False
        """
        if not self.case_sensitive:
            for regex in self.regexes:
                if re.match(regex, incoming_message.text, re.IGNORECASE):
                    return True
        else:
            for regex in self.regexes:
                if re.match(regex, incoming_message.text):
                    return True

        return False

class CommandHook(Hook):
    def __init__(self, user_function, command):
        """
        Create new command hook

        :param user_function: decorated user's function, called
                              when command matching with incoming message
        :param command: str, command text without '!'.
                        For example, 'join'.
        """
        self.type = 'message'
        self.priority = 1

        self.func = user_function
        self.command = command

    def check(self, incoming_message):
        """
        Check, is this message catching for this hook

        :param incoming_message: IncomingMessage object
        :return: True or False
        """
        return incoming_message.text.lstrip().startswith('!' + self.command)

def find_hooks(plugin_module):
    """
    Find hooks in plugin module

    :param plugin_module: module object
    :return: list of Hook objects
    """
    hooks = []
    # Iterating throw plugin elements
    for name in plugin_module.__dict__.keys():
        item = plugin_module.__dict__[name]

        # Hooks are setting '_sheldon_hook' parameters
        # on functions when they decorated.
        if hasattr(item, '_sheldon_hook'):
            hooks.append(item._sheldon_hook)
    return hooks

def message(regexes=[], case_sensitive=False):
    """
    Hook for catching messages, for example:
    "i want cat", "thanks" etc.

    :param regexes: list[string], regular expressions for catching messages
    :param case_sensitive: bool, is bot catching messages with case sensitive
    :return:
    """

    def hook(func):
        def wrapped(message_object, bot_object):
            """
            Wrapper around user's function

            :param message_object: incoming message, Message object
            :param bot_object: Sheldon object
            :return:
            """
            return func(message_object, bot_object)

        wrapped._sheldon_hook = MessageHook(wrapped, regexes, case_sensitive)
        return wrapped

    return hook

def command(command_text):
    """
    Hook for catching commands, for example:
    "!asana", "!deploy sheldon" etc.

    :param command_text: str, command text without '!'. For example, 'join'.
    :return:
    """

    def hook(func):
        def wrapped(message_object, bot_object):
            """
            Wrapper around user's function

            :param message_object: incoming message, Message object
            :param bot_object: Sheldon object
            :return:
            """
            return func(message_object, bot_object)

        wrapped._sheldon_hook = CommandHook(wrapped, command_text)
        return wrapped

    return hook
<commit_msg>Add callback hook<commit_after># -*- coding: utf-8 -*-

"""
Decorators and other functions for hooks.

@author: Seva Zhidkov
@contact: zhidkovseva@gmail.com
@license: The MIT license

Copyright (C) 2015
"""
import sys
import _thread as thread
import re

from sheldon.exceptions import catch_module_errors

class Hook:
    """
    Basic class for hooks
    """
    # Every hook has type. It could be
    # 'command' (command hook), 'message' (message hook),
    # 'interval' (interval hook), 'cron' (cron hook),
    # 'web' (web hook)
    type = ''
    func = None

    # If not one module return True after check,
    # bot sort matches by priority and choose first.
    priority = 0

    @catch_module_errors
    def call(self, incoming_message, bot):
        """
        Run hook

        :param incoming_message: Message object
        :param bot: Sheldon object
        :return:
        """
        thread.start_new_thread(self.func, (
            incoming_message, bot
        ))

class MessageHook(Hook):
    def __init__(self, user_function, regexes, case_sensitive=False):
        """
        Create new message hook

        :param user_function: decorated user's function, called
                              when regular expression matching with
                              incoming message
        :param regexes: regular expression for messages
        :param case_sensitive: bool, is hook catching messages
                                     with case sensitive
        """
        self.type = 'message'
        self.priority = 3

        self.func = user_function

        if type(regexes) == str:
            regexes = [regexes]
        self.regexes = regexes

        self.case_sensitive = case_sensitive

    def check(self, incoming_message):
        """
        Check, is this message catching for this hook

        :param incoming_message: IncomingMessage object
        :return: True or False
        """
        if not self.case_sensitive:
            for regex in self.regexes:
                if re.match(regex, incoming_message.text, re.IGNORECASE):
                    return True
        else:
            for regex in self.regexes:
                if re.match(regex, incoming_message.text):
                    return True

        return False

class CommandHook(Hook):
    def __init__(self, user_function, command):
        """
        Create new command hook

        :param user_function: decorated user's function, called
                              when command matching with incoming message
        :param command: str, command text without '!'.
                        For example, 'join'.
        """
        self.type = 'command'
        self.priority = 2

        self.func = user_function
        self.command = command

    def check(self, incoming_message):
        """
        Check, is this message catching for this hook

        :param incoming_message: IncomingMessage object
        :return: True or False
        """
        return incoming_message.text.lstrip().startswith('!' + self.command)

class CallbackHook(Hook):
    def __init__(self, user_function, callback_func):
        """
        Create new function hook.

        :param user_function: decorated user's function, called
                              when command matching with incoming message
        :param callback_func: func, that called with incoming_message and
                              it must return True or False
        """
        self.type = 'callback'
        self.priority = 1

        self.func = user_function
        self.callback_func = callback_func

    def check(self, incoming_message):
        """
        Check, is this message catching for this hook

        :param incoming_message: IncomingMessage object
        :return: True or False
        """
        return self.callback_func.__call__(incoming_message)

def find_hooks(plugin_module):
    """
    Find hooks in plugin module

    :param plugin_module: module object
    :return: list of Hook objects
    """
    hooks = []
    # Iterating throw plugin elements
    for name in plugin_module.__dict__.keys():
        item = plugin_module.__dict__[name]

        # Hooks are setting '_sheldon_hook' parameters
        # on functions when they decorated.
        if hasattr(item, '_sheldon_hook'):
            hooks.append(item._sheldon_hook)
    return hooks

def message(regexes=[], case_sensitive=False):
    """
    Hook for catching messages, for example:
    "i want cat", "thanks" etc.

    :param regexes: list[string], regular expressions for catching messages
    :param case_sensitive: bool, is bot catching messages with case sensitive
    :return:
    """

    def hook(func):
        def wrapped(message_object, bot_object):
            """
            Wrapper around user's function

            :param message_object: incoming message, Message object
            :param bot_object: Sheldon object
            :return:
            """
            return func(message_object, bot_object)

        wrapped._sheldon_hook = MessageHook(wrapped, regexes, case_sensitive)
        return wrapped

    return hook

def command(command_text):
    """
    Hook for catching commands, for example:
    "!asana", "!deploy sheldon" etc.

    :param command_text: str, command text without '!'. For example, 'join'.
    :return:
    """

    def hook(func):
        def wrapped(message_object, bot_object):
            """
            Wrapper around user's function

            :param message_object: incoming message, Message object
            :param bot_object: Sheldon object
            :return:
            """
            return func(message_object, bot_object)

        wrapped._sheldon_hook = CommandHook(wrapped, command_text)
        return wrapped

    return hook

def callback(callback_func):
    """
    Hook for catching anything you want, based on
    your callback function

    :param callback_func: callback func, takes incoming message
                          and returns True or False
    :return:
    """

    def hook(func):
        def wrapped(message_object, bot_object):
            """
            Wrapper around user's function

            :param message_object: incoming message, Message object
            :param bot_object: Sheldon object
            :return:
            """
            return func(message_object, bot_object)

        wrapped._sheldon_hook = CallbackHook(wrapped, callback_func)
        return wrapped

    return hook
<|endoftext|>


<commit_before>from django.conf import settings
from django.forms import MultipleChoiceField, CheckboxSelectMultiple
from haystack.forms import ModelSearchForm, FacetedModelSearchForm

class PreSelectedModelSearchForm(FacetedModelSearchForm):
    possible_facets = MultipleChoiceField(widget=CheckboxSelectMultiple,
                                          choices=())

    def __init__(self, *args, **kwargs):
        """
        If we're in a recognised faceting engine, display and allow faceting.
        """
        super(PreSelectedModelSearchForm, self).__init__(*args, **kwargs)
        engine = getattr(settings, 'HAYSTACK_SEARCH_ENGINE', 'none')
        if engine in ('solr', 'xapian'):
            possible_facets = []
            for k, v in self.searchqueryset.site._field_mapping().items():
                if v['facet_fieldname'] is not None:
                    possible_facets.append(v['facet_fieldname'])
            self.fields['possible_facets'].choices = [(x, x) for x in possible_facets]

    def no_query_found(self):
        """
        When nothing is entered, show everything, because it's a better
        useful default for our usage.
        """
        return self.searchqueryset.all()

    def search(self):
        sqs = super(PreSelectedModelSearchForm, self).search()
        cleaned_data = getattr(self, 'cleaned_data', {})
        to_facet_on = cleaned_data.get('possible_facets', [])
        if len(to_facet_on) > 0:
            for field in to_facet_on:
                sqs = sqs.facet(field)
        return sqs
<commit_msg>fix the facetable fields in Haystack 2.x … also allowed unsupported engines to see fields to facet on, for now.<commit_after>from django.conf import settings
from django.forms import MultipleChoiceField, CheckboxSelectMultiple
from haystack.forms import ModelSearchForm, FacetedModelSearchForm

class PreSelectedModelSearchForm(FacetedModelSearchForm):
    possible_facets = MultipleChoiceField(widget=CheckboxSelectMultiple,
                                          choices=())

    def __init__(self, *args, **kwargs):
        """
        If we're in a recognised faceting engine, display and allow faceting.
        """
        super(PreSelectedModelSearchForm, self).__init__(*args, **kwargs)
        self.fields['possible_facets'].choices = self.configure_faceting()

    def configure_faceting(self):
        try:
            # 2.x
            from haystack import connections
            facet_fields = connections['default'].get_unified_index()._facet_fieldnames
            possible_facets = facet_fields.keys()
        except ImportError as e:
            # 1.x
            possible_facets = []
            for k, v in self.searchqueryset.site._field_mapping().items():
                if v['facet_fieldname'] is not None:
                    possible_facets.append(v['facet_fieldname'])
        return [(x, x) for x in possible_facets]

    def no_query_found(self):
        """
        When nothing is entered, show everything, because it's a better
        useful default for our usage.
        """
        return self.searchqueryset.all()

    def search(self):
        sqs = super(PreSelectedModelSearchForm, self).search()
        cleaned_data = getattr(self, 'cleaned_data', {})
        to_facet_on = cleaned_data.get('possible_facets', [])
        if len(to_facet_on) > 0:
            for field in to_facet_on:
                sqs = sqs.facet(field)
        return sqs
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2018 Aaron LI <aly@aaronly.me>
# MIT License
#

"""
Montage a list of sky patches (e.g., simulated sky maps of radio halos
of 10x10 deg^2) to create a large map (e.g., hemisphere) for OSKAR
simulation to help investigate the far side confusion noise (FSCN).

Configuration file (YAML format)
--------------------------------
nside: 8  # npix=768, pixsize~7.33[deg]

region:
  center: [0.0, -27.0]  # [deg]
  rmin: 10.0  # [deg]
  rmax: 80.0  # [deg]

threshold:
  min: 0.0001  # [K]
  max: null

frequency:
  type: calc
  start: 154.0  # [MHz]
  stop: 162.0
  step: 0.16

input:
  filename: '{dir}/cluster_{freq:06.2f}.fits'
  dirlist: 'dir.list'  # filename of the directory list file
  pixelsize: 20  # [arcsec]

output:
  filename: 'skymodel/cluster_{freq:06.2f}.osm'
  clobber: true
"""

import os
import argparse
import logging
from functools import lru_cache

import yaml
import numpy as np
import healpy as hp
import astropy.units as au
from astropy.io import fits
from astropy.wcs import WCS

logging.basicConfig(level=logging.INFO,
                    format='[%(levelname)s:%(lineno)d] %(message)s')
logger = logging.getLogger()

def get_frequencies(config):
    config_freq = config['frequency']
    if config_freq['type'] == 'custom':
        return np.array(config_freq['frequencies'])
    else:
        start = config_freq['start']
        stop = config_freq['stop']
        step = config_freq['step']
        return np.arange(start, stop+step/2, step)

def central_angle(points, p0):
    """
    Calculate the central angles between the points with respect to the
    reference point (p0) on the sphere.

    Input parameters:
        points: (longitude, latitude) [deg]
            point coordinates, two columns
        p0: (longitude, latitude) [deg]
            coordinate of reference point

    Algorithm:
        (radial, azimuthal, polar): (r, theta, phi)
        central_angle: alpha
        longitude: lambda = theta
        latitude: delta = 90 - phi
        colatitude: phi

        Unit vector:
        \hat{r}_1 = (cos(theta1) sin(phi1), sin(theta1) sin(phi1), cos(phi1))
            = (cos(lambda1) cos(delta1), sin(lambda1) cos(delta1), sin(delta1))
        \hat{r}_2 = (cos(theta2) sin(phi2), sin(theta2) sin(phi2), cos(phi2))
            = (cos(lambda2) cos(delta2), sin(lambda2) cos(delta2), sin(delta2))

        Therefore the angle (alpha) between \hat{r}_1 and \hat{r}_2:
        cos(alpha) = \hat{r}_1 \cdot \hat{r}_2
            = cos(delta1) cos(delta2) cos(lambda1-lambda2)
              + sin(delta1) sin(delta2)

    References:
    [1] Spherical Coordinates - Wolfram MathWorld
        http://mathworld.wolfram.com/SphericalCoordinates.html
        Equation (19)
    [2] Great Circle - Wolfram MathWorld
        http://mathworld.wolfram.com/GreatCircle.html
        Equation (1), (2), (4)
    """
    points = np.deg2rad(points)
    if points.ndim == 1:
        lbd1, delta1 = points[0], points[1]
    elif points.ndim == 2:
        lbd1, delta1 = points[:, 0], points[:, 1]
    else:
        raise ValueError('invalid input points')
    lbd0, delta0 = np.deg2rad(p0)
    dotvalue = (np.cos(delta1) * np.cos(delta0) * np.cos(lbd1-lbd0) +
                np.sin(delta1) * np.sin(delta0))
    alpha = np.arccos(dotvalue)
    return np.rad2deg(alpha)  # [deg]

def get_healpix_coords(nside):
    npix = hp.nside2npix(nside)
    lon, lat = hp.pix2ang(nside, np.arange(npix), lonlat=True)
    return (lon, lat)  # [deg]

class Patches:
    def __init__(self, config):
        config_input = config['input']
        self._filename = config_input['filename']
        self._dirlist = [d.strip()
                         for d in open(config_input['dirlist']).readlines()]

    def get_image(self, freq, i):
        i = i % len(self._dirlist)
        fn = self._filename.format(freq=freq, dir=self._dirlist[i])
        logger.info('Load image: %s' % fn)
        with fits.open(fn) as f:
            return (f[0].data, f[0].header)

class SkyModel:
    def __init__(self, image, freq, pixelsize, p0, min_=1e-4, max_=None):
        self._image = image
        self._freq = freq  # [MHz]
        self._pixelsize = pixelsize  # [arcsec]
        self._p0 = p0  # (lon, lat) [deg]
        self._min = min_  # [K]
        self._max = max_  # [K]

    @property
    def K2JyPixel(self):
        pixarea = (self._pixelsize * au.arcsec) ** 2
        equiv = au.brightness_temperature(pixarea, self._freq*au.MHz)
        return au.K.to(au.Jy, equivalencies=equiv)

    @property
    def mask(self):
        _max = self._max or np.inf
        return ((np.abs(self._image) >= self._min) &
                (np.abs(self._image) <= _max))

    @property
    def wcs(self):
        shape = self._image.shape
        delta = self._pixelsize / 3600.0  # [deg]
        wcs = WCS(naxis=2)
        projection = 'SIN'
        wcs.wcs.ctype = ['RA---'+projection, 'DEC--'+projection]
        wcs.wcs.crval = np.array(self._p0)
        wcs.wcs.crpix = np.array([shape[1], shape[0]]) / 2.0 + 1
        wcs.wcs.cdelt = np.array([-delta, delta])
        return wcs

    @property
    def sky(self):
        shape = self._image.shape
        idx = self.mask.flatten()
        wcs = self.wcs
        x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]))
        pix = np.column_stack([x.flatten()[idx], y.flatten()[idx]])
        world = wcs.wcs_pix2world(pix, 0)
        ra, dec = world[:, 0], world[:, 1]
        flux = self._image.flatten()[idx] * self.K2JyPixel
        return np.column_stack([ra, dec, flux])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Montage sky patches to make a large OSKAR sky model')
    parser.add_argument('config', help='configuration file in YAML format')
    args = parser.parse_args()

    config = yaml.load(open(args.config))
    patches = Patches(config)
    frequencies = get_frequencies(config)  # [MHz]
    nfreq = len(frequencies)
    pcenter = config['region']['center']  # [deg]
    rmin = config['region']['rmin']  # [deg]
    rmax = config['region']['rmax']  # [deg]
    min_ = config['threshold']['min']  # [K]
    max_ = config['threshold']['max']  # [K]
    pixelsize = config['input']['pixelsize']  # [arcsec]
    clobber = config['output']['clobber']

    nside = config['nside']
    logger.info('Nside: %d' % nside)
    resol = hp.nside2resol(nside, arcmin=True)  # [arcmin]
    logger.info('HEALPix resolution: %.2f [arcmin]' % resol)
    imgsize = int(round(resol * 60 / pixelsize))
    logger.info('Image patch size: %d' % imgsize)
    plon, plat = get_healpix_coords(nside)  # [deg]
    npatch = len(plon)

    for i, freq in enumerate(frequencies):
        logger.info('[%d/%d] %.2f[MHz] ...' % (i+1, nfreq, freq))
        outfile = config['output']['filename'].format(freq=freq)
        os.makedirs(os.path.dirname(outfile), exist_ok=True)
        if os.path.exists(outfile) and (not clobber):
            raise FileExistsError(outfile)

        results = []
        jj = 0
        for j, p0 in enumerate(zip(plon, plat)):
            logger.info('[%d/%d|%d] patch @ (%.2f, %.2f)' %
                        (j+1, jj+1, npatch, p0[0], p0[1]))
            if central_angle(p0, pcenter) > rmax:
                logger.info('skip')
                continue

            image, header = patches.get_image(freq, jj)
            skymodel = SkyModel(image[:imgsize, :imgsize],
                                freq, pixelsize=pixelsize,
                                p0=p0, min_=min_, max_=max_)
            sky = skymodel.sky
            points = sky[:, 0:2]  # (lon, lat)
            angles = central_angle(points, pcenter)  # [deg]
            idx = ((angles >= rmin) & (angles <= rmax))
            sky = sky[idx, :]
            logger.info('Source counts: %d' % len(sky))
            results.append(sky)
            jj += 1

        sky = np.row_stack(results)
        logger.info('Total source counts: %d' % len(sky))
        header = ('Frequency = %.2f [MHz]\n' % freq +
                  'Source counts = %d\n\n' % len(sky) +
                  'R.A.[deg]    Dec.[deg]    flux[Jy]')
        logger.info('Writing sky model to file: %s ...' % outfile)
        np.savetxt(outfile, sky, fmt='%.10e, %.10e, %.10e', header=header)
        logger.info("Wrote OSKAR sky model to file: %s" % outfile)
<commit_msg>oskar/montage_skypatch.py: Fix a log message<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2018 Aaron LI <aly@aaronly.me>
# MIT License
#

"""
Montage a list of sky patches (e.g., simulated sky maps of radio halos
of 10x10 deg^2) to create a large map (e.g., hemisphere) for OSKAR
simulation to help investigate the far side confusion noise (FSCN).

Configuration file (YAML format)
--------------------------------
nside: 8  # npix=768, pixsize~7.33[deg]

region:
  center: [0.0, -27.0]  # [deg]
  rmin: 10.0  # [deg]
  rmax: 80.0  # [deg]

threshold:
  min: 0.0001  # [K]
  max: null

frequency:
  type: calc
  start: 154.0  # [MHz]
  stop: 162.0
  step: 0.16

input:
  filename: '{dir}/cluster_{freq:06.2f}.fits'
  dirlist: 'dir.list'  # filename of the directory list file
  pixelsize: 20  # [arcsec]

output:
  filename: 'skymodel/cluster_{freq:06.2f}.osm'
  clobber: true
"""

import os
import argparse
import logging
from functools import lru_cache

import yaml
import numpy as np
import healpy as hp
import astropy.units as au
from astropy.io import fits
from astropy.wcs import WCS

logging.basicConfig(level=logging.INFO,
                    format='[%(levelname)s:%(lineno)d] %(message)s')
logger = logging.getLogger()

def get_frequencies(config):
    config_freq = config['frequency']
    if config_freq['type'] == 'custom':
        return np.array(config_freq['frequencies'])
    else:
        start = config_freq['start']
        stop = config_freq['stop']
        step = config_freq['step']
        return np.arange(start, stop+step/2, step)

def central_angle(points, p0):
    """
    Calculate the central angles between the points with respect to the
    reference point (p0) on the sphere.

    Input parameters:
        points: (longitude, latitude) [deg]
            point coordinates, two columns
        p0: (longitude, latitude) [deg]
            coordinate of reference point

    Algorithm:
        (radial, azimuthal, polar): (r, theta, phi)
        central_angle: alpha
        longitude: lambda = theta
        latitude: delta = 90 - phi
        colatitude: phi

        Unit vector:
        \hat{r}_1 = (cos(theta1) sin(phi1), sin(theta1) sin(phi1), cos(phi1))
            = (cos(lambda1) cos(delta1), sin(lambda1) cos(delta1), sin(delta1))
        \hat{r}_2 = (cos(theta2) sin(phi2), sin(theta2) sin(phi2), cos(phi2))
            = (cos(lambda2) cos(delta2), sin(lambda2) cos(delta2), sin(delta2))

        Therefore the angle (alpha) between \hat{r}_1 and \hat{r}_2:
        cos(alpha) = \hat{r}_1 \cdot \hat{r}_2
            = cos(delta1) cos(delta2) cos(lambda1-lambda2)
              + sin(delta1) sin(delta2)

    References:
    [1] Spherical Coordinates - Wolfram MathWorld
        http://mathworld.wolfram.com/SphericalCoordinates.html
        Equation (19)
    [2] Great Circle - Wolfram MathWorld
        http://mathworld.wolfram.com/GreatCircle.html
        Equation (1), (2), (4)
    """
    points = np.deg2rad(points)
    if points.ndim == 1:
        lbd1, delta1 = points[0], points[1]
    elif points.ndim == 2:
        lbd1, delta1 = points[:, 0], points[:, 1]
    else:
        raise ValueError('invalid input points')
    lbd0, delta0 = np.deg2rad(p0)
    dotvalue = (np.cos(delta1) * np.cos(delta0) * np.cos(lbd1-lbd0) +
                np.sin(delta1) * np.sin(delta0))
    alpha = np.arccos(dotvalue)
    return np.rad2deg(alpha)  # [deg]

def get_healpix_coords(nside):
    npix = hp.nside2npix(nside)
    lon, lat = hp.pix2ang(nside, np.arange(npix), lonlat=True)
    return (lon, lat)  # [deg]

class Patches:
    def __init__(self, config):
        config_input = config['input']
        self._filename = config_input['filename']
        self._dirlist = [d.strip()
                         for d in open(config_input['dirlist']).readlines()]

    def get_image(self, freq, i):
        i = i % len(self._dirlist)
        fn = self._filename.format(freq=freq, dir=self._dirlist[i])
        logger.info('Load image: %s' % fn)
        with fits.open(fn) as f:
            return (f[0].data, f[0].header)

class SkyModel:
    def __init__(self, image, freq, pixelsize, p0, min_=1e-4, max_=None):
        self._image = image
        self._freq = freq  # [MHz]
        self._pixelsize = pixelsize  # [arcsec]
        self._p0 = p0  # (lon, lat) [deg]
        self._min = min_  # [K]
        self._max = max_  # [K]

    @property
    def K2JyPixel(self):
        pixarea = (self._pixelsize * au.arcsec) ** 2
        equiv = au.brightness_temperature(pixarea, self._freq*au.MHz)
        return au.K.to(au.Jy, equivalencies=equiv)

    @property
    def mask(self):
        _max = self._max or np.inf
        return ((np.abs(self._image) >= self._min) &
                (np.abs(self._image) <= _max))

    @property
    def wcs(self):
        shape = self._image.shape
        delta = self._pixelsize / 3600.0  # [deg]
        wcs = WCS(naxis=2)
        projection = 'SIN'
        wcs.wcs.ctype = ['RA---'+projection, 'DEC--'+projection]
        wcs.wcs.crval = np.array(self._p0)
        wcs.wcs.crpix = np.array([shape[1], shape[0]]) / 2.0 + 1
        wcs.wcs.cdelt = np.array([-delta, delta])
        return wcs

    @property
    def sky(self):
        shape = self._image.shape
        idx = self.mask.flatten()
        wcs = self.wcs
        x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]))
        pix = np.column_stack([x.flatten()[idx], y.flatten()[idx]])
        world = wcs.wcs_pix2world(pix, 0)
        ra, dec = world[:, 0], world[:, 1]
        flux = self._image.flatten()[idx] * self.K2JyPixel
        return np.column_stack([ra, dec, flux])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Montage sky patches to make a large OSKAR sky model')
    parser.add_argument('config', help='configuration file in YAML format')
    args = parser.parse_args()

    config = yaml.load(open(args.config))
    patches = Patches(config)
    frequencies = get_frequencies(config)  # [MHz]
    nfreq = len(frequencies)
    pcenter = config['region']['center']  # [deg]
    rmin = config['region']['rmin']  # [deg]
    rmax = config['region']['rmax']  # [deg]
    min_ = config['threshold']['min']  # [K]
    max_ = config['threshold']['max']  # [K]
    pixelsize = config['input']['pixelsize']  # [arcsec]
    clobber = config['output']['clobber']

    nside = config['nside']
    logger.info('Nside: %d' % nside)
    resol = hp.nside2resol(nside, arcmin=True)  # [arcmin]
    logger.info('HEALPix resolution: %.2f [arcmin]' % resol)
    imgsize = int(round(resol * 60 / pixelsize))
    logger.info('Image patch size: %d' % imgsize)
    plon, plat = get_healpix_coords(nside)  # [deg]
    npatch = len(plon)

    for i, freq in enumerate(frequencies):
        logger.info('[%d/%d] %.2f[MHz] ...' % (i+1, nfreq, freq))
        outfile = config['output']['filename'].format(freq=freq)
        os.makedirs(os.path.dirname(outfile), exist_ok=True)
        if os.path.exists(outfile) and (not clobber):
            raise FileExistsError(outfile)

        results = []
        jj = 0
        for j, p0 in enumerate(zip(plon, plat)):
            logger.info('[%d/%d|%d] patch @ (%.2f, %.2f)' %
                        (j+1, npatch, jj+1, p0[0], p0[1]))
            if central_angle(p0, pcenter) > rmax:
                logger.info('skip')
                continue

            image, header = patches.get_image(freq, jj)
            skymodel = SkyModel(image[:imgsize, :imgsize],
                                freq, pixelsize=pixelsize,
                                p0=p0, min_=min_, max_=max_)
            sky = skymodel.sky
            points = sky[:, 0:2]  # (lon, lat)
            angles = central_angle(points, pcenter)  # [deg]
            idx = ((angles >= rmin) & (angles <= rmax))
            sky = sky[idx, :]
            logger.info('Source counts: %d' % len(sky))
            results.append(sky)
            jj += 1

        sky = np.row_stack(results)
        logger.info('Total source counts: %d' % len(sky))
        header = ('Frequency = %.2f [MHz]\n' % freq +
                  'Source counts = %d\n\n' % len(sky) +
                  'R.A.[deg]    Dec.[deg]    flux[Jy]')
        logger.info('Writing sky model to file: %s ...' % outfile)
        np.savetxt(outfile, sky, fmt='%.10e, %.10e, %.10e', header=header)
        logger.info("Wrote OSKAR sky model to file: %s" % outfile)
<|endoftext|>


<commit_before>import dj_database_url
import os

from .base import *

DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = [('znotdead', 'zhirafchik@gmail.com')]

DATABASES['default'] = dj_database_url.config()

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATIC_URL = '/static/'

ALLOWED_HOSTS = [".herokuapp.com", ]

REDIS_POOL = redis.ConnectionPool.from_url(os.environ.get('REDISCLOUD_URL'))
REDIS_CONN = redis.Redis(connection_pool=REDIS_POOL)
<commit_msg>#16 redirect to https<commit_after>import dj_database_url
import os

from .base import *

DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = [('znotdead', 'zhirafchik@gmail.com')]

DATABASES['default'] = dj_database_url.config()

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATIC_URL = '/static/'

ALLOWED_HOSTS = [".herokuapp.com", ]

REDIS_POOL = redis.ConnectionPool.from_url(os.environ.get('REDISCLOUD_URL'))
REDIS_CONN = redis.Redis(connection_pool=REDIS_POOL)

if not DEBUG:
    SECURE_SSL_REDIRECT = True
<|endoftext|>


<commit_before>import ast
import html
import inspect
import json
import os
import traceback
from collections import defaultdict, Sequence, Set, Mapping, deque, namedtuple
from datetime import datetime
from functools import partial
from itertools import chain, islice
from threading import Lock
from uuid import uuid4

from asttokens import ASTTokens

from birdseye.cheap_repr import cheap_repr
from birdseye.db import Function, Call, session, db_consumer
from birdseye.tracer import TreeTracerBase, loops, TracedFile
from birdseye.utils import safe_qualname, correct_type, exception_string, dummy_namespace

CodeInfo = namedtuple('CodeInfo', 'db_func traced_file')

class BirdsEye(TreeTracerBase):
    def __init__(self):
        super().__init__()
        self._code_infos = {}

    def parse_extra(self, root, source, filename):
        for node in ast.walk(root):
            node._loops = loops(node)

    def compile(self, source, filename):
        traced_file = super(BirdsEye, self).compile(source, filename)
        traced_file.tokens = ASTTokens(source, tree=traced_file.root)
        return traced_file

    def before_stmt(self, node, frame):
        if frame.f_code not in self._code_infos:
            return
        if isinstance(node.parent, (ast.For, ast.While)) and node is node.parent.body[0]:
            iteration = self.stack[frame].iteration
            for i, loop_node in enumerate(node._loops):
                loop = iteration.loops[loop_node._tree_index]
                if i == len(node._loops) - 1:
                    loop.append(Iteration())
                else:
                    iteration = loop.last()
        self._set_node_value(node, frame, True)

    def before_expr(self, node, frame):
        self.stack[frame].inner_call = None

    def after_expr(self, node, frame, value):
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        if is_obvious_builtin(node, frame_info):
            return

        value = expand(value, level=max(1, 3 - len(node._loops)))
        if frame_info.inner_call:
            value.insert(2, {'inner_call': frame_info.inner_call})
            frame_info.inner_call = None
        self._set_node_value(node, frame, value)

    def _set_node_value(self, node, frame, value):
        iteration = self.stack[frame].iteration
        for i, loop_node in enumerate(node._loops):
            loop = iteration.loops[loop_node._tree_index]
            iteration = loop.last()
        iteration.vals[node._tree_index] = value

    def after_stmt(self, node, frame, exc_value, exc_traceback):
        if frame.f_code not in self._code_infos:
            return
        expression_stack = self.stack[frame].expression_stack
        if expression_stack:
            self._set_node_value(
                expression_stack[-1], frame,
                [exception_string(exc_value), -1])

    def enter_call(self, enter_info):
        frame = enter_info.current_frame
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        frame_info.start_time = datetime.now()
        frame_info.iteration = Iteration()
        arg_info = inspect.getargvalues(frame)
        arg_names = chain(arg_info[0], arg_info[1:3])
        f_locals = arg_info[3].copy()
        arguments = [(name, f_locals.pop(name)) for name in arg_names if name] + list(f_locals.items())
        frame_info.arguments = json.dumps([[k, cheap_repr(v)] for k, v in arguments])
        self.stack.get(frame.f_back, dummy_namespace).inner_call = frame_info.call_id = uuid4().hex

    def exit_call(self, exit_info):
        frame = exit_info.current_frame
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        top_iteration = frame_info.iteration

        loop_iterations = top_iteration.extract_iterations()['loops']

        node_values = _deep_dict()

        def extract_values(iteration, path):
            for k, v in iteration.vals.items():
                full_path = (k,) + path
                d = node_values
                for path_k in full_path[:-1]:
                    d = d[path_k]
                d[full_path[-1]] = v

            for loop in iteration.loops.values():
                for i, iteration in enumerate(loop):
                    extract_values(iteration, path + (i,))

        extract_values(top_iteration, ())

        db_func = self._code_infos[frame.f_code].db_func
        exc = exit_info.exc_value
        if exc:
            traceback_str = ''.join(traceback.format_exception(type(exc), exc, exit_info.exc_tb))
            exception = exception_string(exc)
        else:
            traceback_str = exception = None

        @db_consumer
        def save_call():
            call = Call(id=frame_info.call_id,
                        function=db_func,
                        arguments=frame_info.arguments,
                        return_value=cheap_repr(exit_info.return_value),
                        exception=exception,
                        traceback=traceback_str,
                        data=json.dumps(
                            dict(
                                node_values=node_values,
                                loop_iterations=loop_iterations,
                                type_names=type_registry.names(),
                                num_special_types=type_registry.num_special_types,
                            ),
                            separators=(',', ':')
                        ),
                        start_time=frame_info.start_time)
            session.add(call)
            session.commit()

    def __call__(self, func):
        new_func = super(BirdsEye, self).__call__(func)
        code_info = self._code_infos.get(new_func.__code__)
        if code_info:
            return new_func
        lines, start_lineno = inspect.getsourcelines(func)
        end_lineno = start_lineno + len(lines)
        name = safe_qualname(func)
        filename = os.path.abspath(inspect.getsourcefile(func))

        traced_file = new_func.traced_file
        traced_file.root._depth = 0
        for node in ast.walk(traced_file.root):
            for child in ast.iter_child_nodes(node):
                child._depth = node._depth + 1

        positions = []
        node_loops = {}
        for node in traced_file.nodes:
            if isinstance(node, ast.expr):
                node_type = 'expr'
                if (isinstance(node, (ast.Num, ast.Str)) or
                        isinstance(getattr(node, 'ctx', None),
                                   (ast.Store, ast.Del))):
                    continue
            elif isinstance(node, (ast.While, ast.For)):
                node_type = 'loop'
            elif isinstance(node, ast.stmt):
                node_type = 'stmt'
            else:
                continue
            assert isinstance(node, ast.AST)
            if not start_lineno <= node.lineno <= end_lineno:
                continue
            if (isinstance(node, ast.UnaryOp) and
                    isinstance(node.op, (ast.UAdd, ast.USub)) and
                    isinstance(node.operand, ast.Num)):
                continue
            start, end = traced_file.tokens.get_text_range(node)
            if start == end == 0:
                continue
            positions.append((start, 1, node._depth,
                              '' % (node._tree_index, node_type)))
            positions.append((end, 0, node._depth, ''))
            if node._loops:
                node_loops[node._tree_index] = [n._tree_index for n in node._loops]
        positions.append((len(traced_file.source), None, None, ''))
        positions.sort()

        html_body = []
        start = 0
        for pos, _, _, part in positions:
            html_body.append(html.escape(traced_file.source[start:pos]))
            html_body.append(part)
            start = pos
        html_body = ''.join(html_body)
        html_body = '\n'.join(html_body.split('\n')[start_lineno - 1:end_lineno - 1])

        db_args = dict(file=filename,
                       name=name,
                       html_body=html_body,
                       lineno=start_lineno,
                       data=json.dumps(
                           dict(
                               node_loops=node_loops,
                           ),
                           sort_keys=True,
                       ))

        @db_consumer.wait
        def save_func():
            db_func = session.query(Function).filter_by(**db_args).one_or_none()
            if not db_func:
                db_func = Function(**db_args)
                session.add(db_func)
                session.commit()
            self._code_infos[new_func.__code__] = CodeInfo(db_func, traced_file)

        return new_func

def _deep_dict():
    return defaultdict(_deep_dict)

class IterationList(object):
    side_len = 3

    def __init__(self):
        self.start = []
        self.end = deque(maxlen=self.side_len)
        self.length = 0

    def append(self, x):
        if self.length < self.side_len:
            self.start.append(x)
        else:
            self.end.append(x)
        x.index = self.length
        self.length += 1

    def __iter__(self):
        return chain(self.start, self.end)

    def last(self):
        if self.end:
            return self.end[-1]
        else:
            return self.start[-1]

class Iteration(object):
    def __init__(self):
        self.vals = {}
        self.loops = defaultdict(IterationList)
        self.index = None

    def extract_iterations(self):
        return {
            'index': self.index,
            'loops': {
                k: [it.extract_iterations() for it in v]
                for k, v in self.loops.items()
            }
        }

class TypeRegistry(object):
    def __init__(self):
        self.lock = Lock()
        self.data = defaultdict(lambda: len(self.data))
        basic_types = [type(None), bool, int, float, complex]  # TODO long, unicode
        special_types = basic_types + [list, dict, tuple, set, frozenset, str, bytes]
        self.num_basic_types = len(basic_types)
        self.num_special_types = len(special_types)
        for t in special_types:
            _ = self.data[t]

    def __getitem__(self, item):
        t = correct_type(item)
        with self.lock:
            return self.data[t]

    def names(self):
        rev = dict((v, k) for k, v in self.data.items())
        return [safe_qualname(rev[i]) for i in range(len(rev))]

type_registry = TypeRegistry()

def expand(val, level=3):
    type_index = type_registry[val]
    result = [cheap_repr(val), type_index]
    if type_index < type_registry.num_basic_types or level == 0:
        return result
    exp = partial(expand, level=level - 1)

    # noinspection PyBroadException
    try:
        length = len(val)
    except:
        pass
    else:
        result += ['len() = %s' % length]

    if isinstance(val, (str, bytes, range)):  # TODO unicode, xrange
        return result
    if isinstance(val, Sequence):
        if len(val) <= 8:
            indices = range(len(val))
        else:
            indices = chain(range(3), range(len(val) - 3, len(val)))
        for i in indices:
            result += [(str(i), exp(val[i]))]
    elif isinstance(val, Mapping):
        for k, v in islice(val.items(), 10):  # TODO iteritems
            result += [(cheap_repr(k), exp(v))]
    elif isinstance(val, Set):
        for i, v in enumerate(islice(val, 6)):
            result += [('<%s>' % i, exp(v))]

    d = getattr(val, '__dict__', None)
    if d:
        for k, v in islice(d.items(), 50):  # TODO iteritems
            if isinstance(v, TracedFile):
                continue
            result += [(str(k), exp(v))]
    else:
        slots = getattr(val, '__slots__', None)
        if slots:
            for s in slots:
                try:
                    attr = getattr(val, s)
                except AttributeError:
                    pass
                else:
                    result += [(str(s), exp(attr))]
    return result

def is_obvious_builtin(node, frame_info):
    try:
        value = frame_info.expression_values[node]
        # noinspection PyUnresolvedReferences
        is_top_level_builtin = ((isinstance(node, ast.Name) and
                                 node.id in __builtins__ and
                                 __builtins__[node.id] is value) or
                                isinstance(node, ast.NameConstant))
        if is_top_level_builtin:
            return True
            # if not isinstance(node, ast.Attribute):
            #     return False
            # name = node.attr
            # if name != value.__name__:
            #     return False
            # if value.__self__ is not frame_info.expression_values[node.value]:
            #     return False
            # TODO complete for methods
    except AttributeError:
        # return False
        raise
<commit_msg>Split up list comprehensions on same line<commit_after>import ast
import html
import inspect
import json
import os
import traceback
from collections import defaultdict, Sequence, Set, Mapping, deque, namedtuple
from datetime import datetime
from functools import partial
from itertools import chain, islice
from threading import Lock
from uuid import uuid4

from asttokens import ASTTokens
from littleutils import group_by_key_func

from birdseye.cheap_repr import cheap_repr
from birdseye.db import Function, Call, session, db_consumer
from birdseye.tracer import TreeTracerBase, loops, TracedFile
from birdseye.utils import safe_qualname, correct_type, exception_string, dummy_namespace

CodeInfo = namedtuple('CodeInfo', 'db_func traced_file')

class BirdsEye(TreeTracerBase):
    def __init__(self):
        super().__init__()
        self._code_infos = {}

    def parse_extra(self, root, source, filename):
        for node in ast.walk(root):
            node._loops = loops(node)

    def compile(self, source, filename):
        traced_file = super(BirdsEye, self).compile(source, filename)
        traced_file.tokens = ASTTokens(source, tree=traced_file.root)
        return traced_file

    def before_stmt(self, node, frame):
        if frame.f_code not in self._code_infos:
            return
        if isinstance(node.parent, (ast.For, ast.While)) and node is node.parent.body[0]:
            iteration = self.stack[frame].iteration
            for i, loop_node in enumerate(node._loops):
                loop = iteration.loops[loop_node._tree_index]
                if i == len(node._loops) - 1:
                    loop.append(Iteration())
                else:
                    iteration = loop.last()
        self._set_node_value(node, frame, True)

    def before_expr(self, node, frame):
        self.stack[frame].inner_call = None

    def after_expr(self, node, frame, value):
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        if is_obvious_builtin(node, frame_info):
            return

        value = expand(value, level=max(1, 3 - len(node._loops)))
        if frame_info.inner_call:
            value.insert(2, {'inner_call': frame_info.inner_call})
            frame_info.inner_call = None
        self._set_node_value(node, frame, value)

    def _set_node_value(self, node, frame, value):
        iteration = self.stack[frame].iteration
        for i, loop_node in enumerate(node._loops):
            loop = iteration.loops[loop_node._tree_index]
            iteration = loop.last()
        iteration.vals[node._tree_index] = value

    def after_stmt(self, node, frame, exc_value, exc_traceback):
        if frame.f_code not in self._code_infos:
            return
        expression_stack = self.stack[frame].expression_stack
        if expression_stack:
            self._set_node_value(
                expression_stack[-1], frame,
                [exception_string(exc_value), -1])

    def enter_call(self, enter_info):
        frame = enter_info.current_frame
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        frame_info.start_time = datetime.now()
        frame_info.iteration = Iteration()
        arg_info = inspect.getargvalues(frame)
        arg_names = chain(arg_info[0], arg_info[1:3])
        f_locals = arg_info[3].copy()
        arguments = [(name, f_locals.pop(name)) for name in arg_names if name] + list(f_locals.items())
        frame_info.arguments = json.dumps([[k, cheap_repr(v)] for k, v in arguments])
        self.stack.get(frame.f_back, dummy_namespace).inner_call = frame_info.call_id = uuid4().hex

    def exit_call(self, exit_info):
        frame = exit_info.current_frame
        if frame.f_code not in self._code_infos:
            return
        frame_info = self.stack[frame]
        top_iteration = frame_info.iteration

        loop_iterations = top_iteration.extract_iterations()['loops']

        node_values = _deep_dict()

        def extract_values(iteration, path):
            for k, v in iteration.vals.items():
                full_path = (k,) + path
                d = node_values
                for path_k in full_path[:-1]:
                    d = d[path_k]
                d[full_path[-1]] = v

            for loop in iteration.loops.values():
                for i, iteration in enumerate(loop):
                    extract_values(iteration, path + (i,))

        extract_values(top_iteration, ())

        db_func = self._code_infos[frame.f_code].db_func
        exc = exit_info.exc_value
        if exc:
            traceback_str = ''.join(traceback.format_exception(type(exc), exc, exit_info.exc_tb))
            exception = exception_string(exc)
        else:
            traceback_str = exception = None

        @db_consumer
        def save_call():
            call = Call(id=frame_info.call_id,
                        function=db_func,
                        arguments=frame_info.arguments,
                        return_value=cheap_repr(exit_info.return_value),
                        exception=exception,
                        traceback=traceback_str,
                        data=json.dumps(
                            dict(
                                node_values=node_values,
                                loop_iterations=loop_iterations,
                                type_names=type_registry.names(),
                                num_special_types=type_registry.num_special_types,
                            ),
                            separators=(',', ':')
                        ),
                        start_time=frame_info.start_time)
            session.add(call)
            session.commit()

    def __call__(self, func):
        new_func = super(BirdsEye, self).__call__(func)
        code_info = self._code_infos.get(new_func.__code__)
        if code_info:
            return new_func
        lines, start_lineno = inspect.getsourcelines(func)
        end_lineno = start_lineno + len(lines)
        name = safe_qualname(func)
        filename = os.path.abspath(inspect.getsourcefile(func))

        traced_file = new_func.traced_file
        traced_file.root._depth = 0
        for node in ast.walk(traced_file.root):
            for child in ast.iter_child_nodes(node):
                child._depth = node._depth + 1

        positions = []
        node_loops = {}
        for node in traced_file.nodes:
            if isinstance(node, ast.expr):
                node_type = 'expr'
                if (isinstance(node, (ast.Num, ast.Str)) or
                        isinstance(getattr(node, 'ctx', None),
                                   (ast.Store, ast.Del))):
                    continue
            elif isinstance(node, (ast.While, ast.For)):
                node_type = 'loop'
            elif isinstance(node, ast.stmt):
                node_type = 'stmt'
            else:
                continue
            assert isinstance(node, ast.AST)
            if not start_lineno <= node.lineno <= end_lineno:
                continue
            if (isinstance(node, ast.UnaryOp) and
                    isinstance(node.op, (ast.UAdd, ast.USub)) and
                    isinstance(node.operand, ast.Num)):
                continue
            start, end = traced_file.tokens.get_text_range(node)
            if start == end == 0:
                continue
            positions.append((start, 1, node._depth,
                              '' % (node._tree_index, node_type)))
            positions.append((end, 0, node._depth, ''))
            if node._loops:
                node_loops[node._tree_index] = [n._tree_index for n in node._loops]

        comprehensions = group_by_key_func([comp for comp in traced_file.nodes
                                            if isinstance(comp, ast.comprehension)],
                                           lambda c: c.first_token.line)

        def get_start(n):
            return traced_file.tokens.get_text_range(n)[0]

        for comp_list in comprehensions.values():
            prev_start = None
            for comp in sorted(comp_list, key=lambda c: c.first_token.startpos):
                if comp is comp.parent.generators[0]:
                    start = get_start(comp.parent)
                    if prev_start is not None and start < prev_start:
                        start = get_start(comp)
                else:
                    start = get_start(comp)
                if prev_start is not None:
                    positions.append((start, 1, 0, '\n '))
                    end_lineno += 1
                prev_start = start

        positions.append((len(traced_file.source), None, None, ''))
        positions.sort()

        html_body = []
        start = 0
        for pos, _, _, part in positions:
            html_body.append(html.escape(traced_file.source[start:pos]))
            html_body.append(part)
            start = pos
        html_body = ''.join(html_body)
        html_body = '\n'.join(html_body.split('\n')[start_lineno - 1:end_lineno - 1])

        db_args = dict(file=filename,
                       name=name,
                       html_body=html_body,
                       lineno=start_lineno,
                       data=json.dumps(
                           dict(
                               node_loops=node_loops,
                           ),
                           sort_keys=True,
                       ))

        @db_consumer.wait
        def save_func():
            db_func = session.query(Function).filter_by(**db_args).one_or_none()
            if not db_func:
                db_func = Function(**db_args)
                session.add(db_func)
                session.commit()
            self._code_infos[new_func.__code__] = CodeInfo(db_func, traced_file)

        return new_func

def _deep_dict():
    return defaultdict(_deep_dict)

class IterationList(object):
    side_len = 3

    def __init__(self):
        self.start = []
        self.end = deque(maxlen=self.side_len)
        self.length = 0

    def append(self, x):
        if self.length < self.side_len:
            self.start.append(x)
        else:
            self.end.append(x)
        x.index = self.length
        self.length += 1

    def __iter__(self):
        return chain(self.start, self.end)

    def last(self):
        if self.end:
            return self.end[-1]
        else:
            return self.start[-1]

class Iteration(object):
    def __init__(self):
        self.vals = {}
        self.loops = defaultdict(IterationList)
        self.index = None

    def extract_iterations(self):
        return {
            'index': self.index,
            'loops': {
                k: [it.extract_iterations() for it in v]
                for k, v in self.loops.items()
            }
        }

class TypeRegistry(object):
    def __init__(self):
        self.lock = Lock()
        self.data = defaultdict(lambda: len(self.data))
        basic_types = [type(None), bool, int, float, complex]  # TODO long, unicode
        special_types = basic_types + [list, dict, tuple, set, frozenset, str, bytes]
        self.num_basic_types = len(basic_types)
        self.num_special_types = len(special_types)
        for t in special_types:
            _ = self.data[t]

    def __getitem__(self, item):
        t = correct_type(item)
        with self.lock:
            return self.data[t]

    def names(self):
        rev = dict((v, k) for k, v in self.data.items())
        return [safe_qualname(rev[i]) for i in range(len(rev))]

type_registry = TypeRegistry()

def expand(val, level=3):
    type_index = type_registry[val]
    result = [cheap_repr(val), type_index]
    if type_index < type_registry.num_basic_types or level == 0:
        return result
    exp = partial(expand, level=level - 1)

    # noinspection PyBroadException
    try:
        length = len(val)
    except:
        pass
    else:
        result += ['len() = %s' % length]

    if isinstance(val, (str, bytes, range)):  # TODO unicode, xrange
        return result
    if isinstance(val, Sequence):
        if len(val) <= 8:
            indices = range(len(val))
        else:
            indices = chain(range(3), range(len(val) - 3, len(val)))
        for i in indices:
            result += [(str(i), exp(val[i]))]
    elif isinstance(val, Mapping):
        for k, v in islice(val.items(), 10):  # TODO iteritems
            result += [(cheap_repr(k), exp(v))]
    elif isinstance(val, Set):
        for i, v in enumerate(islice(val, 6)):
            result += [('<%s>' % i, exp(v))]

    d = getattr(val, '__dict__', None)
    if d:
        for k, v in islice(d.items(), 50):  # TODO iteritems
            if isinstance(v, TracedFile):
                continue
            result += [(str(k), exp(v))]
    else:
        slots = getattr(val, '__slots__', None)
        if slots:
            for s in slots:
                try:
                    attr = getattr(val, s)
                except AttributeError:
                    pass
                else:
                    result += [(str(s), exp(attr))]
    return result

def is_obvious_builtin(node, frame_info):
    try:
        value = frame_info.expression_values[node]
        # noinspection PyUnresolvedReferences
        is_top_level_builtin = ((isinstance(node, ast.Name) and
                                 node.id in __builtins__ and
                                 __builtins__[node.id] is value) or
                                isinstance(node, ast.NameConstant))
        if is_top_level_builtin:
            return True
            # if not isinstance(node, ast.Attribute):
            #     return False
            # name = node.attr
            # if name != value.__name__:
            #     return False
            # if value.__self__ is not frame_info.expression_values[node.value]:
            #     return False
            # TODO complete for methods
    except AttributeError:
        # return False
        raise
<|endoftext|>