<commit_before># videoposter.py is reddit bot that scrapes a website to find
# the last video posted on it and creates a reddit submission
# with a link to the video, then it posts the youtube channel
# mirror as a comment in the submission.

# Made in Python 3.4.3, by /u/MorrisCasper and /u/twistitup.

import requests, time, praw, oaux, traceback
# install with "pip install beautifulsoup4"
from bs4 import BeautifulSoup
# install with "pip install --upgrade google-api-python-client"
from apiclient.discovery import build
from difflib import SequenceMatcher

### USER CONFIGURATION ###

SLEEP_TIME = 300 # Time to sleep between requesting the website in seconds
SUBREDDIT = "TwoBestFriendsPlay" # Without /r/
YOUTUBE_API_KEY = ""
HIGH_SIMILARITY = 0.99
LOW_SIMILARITY = 0.95

### END USER CONFIGURATION ###

# Returns the link and title of the lastest video on the SBFP website.
def get_latest_video():
    r = requests.get("http://superbestfriendsplay.com/category/videos/", "lxml")
    soup = BeautifulSoup(r.content, "html.parser")
    latest_list_item = soup.find("li", {"class": "archiveitem"})
    link = latest_list_item.find("div", {"class": "archivetitle"}).find("a").get("href")
    title = latest_list_item.find("div", {"class": "archivetitle"}).find("a").text
    return link, title

# Compares two strings and returns a float from 0.0 to 1.0 depending on
# the similarity of the strings.
def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()

# Returns whether or not the youtube video is found and the comment is posted.
def postYoutubeComment(title_ws, submission_posted, similarity):
    service = build("youtube", "v3", developerKey=YOUTUBE_API_KEY)
    playlistItems = service.playlistItems()
    results = 50 # Maximum allowed number of requested videos
    request = playlistItems.list(
        part="snippet", playlistId="UU84X0epDRFdTrybxEX8ZWkA",
        maxResults=results)
    uploads = request.execute()
    for i in range(0, results):
        iterationTitle = uploads["items"][i]["snippet"]["title"]
        # Compare how similar the two titles are. If it's almost
        # certain that they are the same video, post the comment
        if similar(iterationTitle, title_ws) > similarity:
            print("Video title on youtube channel: " + iterationTitle)
            print("Posting youtube url comment")
            videoId = (uploads["items"][i]["snippet"]["resourceId"]["videoId"])
            videoURL = "https://www.youtube.com/watch?v=" + videoId
            submission_posted.add_comment(
                "Here is the video on Youtube! " + videoURL)
            print(
                "Submission and comment posted. Sleeping",
                SLEEP_TIME, "seconds.")
            return True
    return False

if __name__ == "__main__":
    r = oaux.login()
    subreddit = r.get_subreddit(SUBREDDIT)
    latest_link_posted = ""
    while True:
        try:
            link_ws, title_ws = get_latest_video() # Latest link and title from website
            print("Latest video on website: " + title_ws)
            if link_ws != latest_link_posted: # If link_ws not already posted
                latest_link = link_ws
                print("Posting latest video")
                submission_posted = subreddit.submit(title_ws, url = link_ws)
                # submission for testing purposes
                # r.get_submission(submission_id = '3h5hzr')
                if postYoutubeComment(
                        title_ws, submission_posted, HIGH_SIMILARITY) == False:
                    print(
                        "Submission posted but not comment.",
                        "Could not find appropriate youtube video.",
                        "Trying again with a lower similarity ratio.")
                    if postYoutubeComment(
                            title_ws, submission_posted,
                            LOW_SIMILARITY) == False:
                        print(
                            "Still couldn't find youtube video,",
                            "please find it and comment manually.",
                            "Sleeping", SLEEP_TIME, "seconds")
            else:
                print(
                    "Not posting because I already posted. Sleeping",
                    SLEEP_TIME, seconds)
            time.sleep(SLEEP_TIME)
        except KeyboardInterrupt:
            print("Shutting down.")
            break
        except praw.errors.HTTPException as e:
            exc = e._raw
            print("Something bad happened! HTTPError", exc.status_code)
            if exc.status_code == 503:
                print("Let's wait til reddit comes back! Sleeping",
                    SLEEP_TIME, "seconds.")
                time.sleep(SLEEP_TIME)
        except Exception as e:
            print("Something bad happened!", e)
            traceback.print_exc()
            print("Waiting", SLEEP_TIME, "seconds.")
            time.sleep(SLEEP_TIME)
<commit_msg>Simplified exception handling.<commit_after># videoposter.py is reddit bot that scrapes a website to find
# the last video posted on it and creates a reddit submission
# with a link to the video, then it posts the youtube channel
# mirror as a comment in the submission.

# Made in Python 3.4.3, by /u/MorrisCasper and /u/twistitup.

import requests, time, praw, oaux, traceback
# install with "pip install beautifulsoup4"
from bs4 import BeautifulSoup
# install with "pip install --upgrade google-api-python-client"
from apiclient.discovery import build
from difflib import SequenceMatcher

### USER CONFIGURATION ###

SLEEP_TIME = 300 # Time to sleep between requesting the website in seconds
SUBREDDIT = "TwoBestFriendsPlay" # Without /r/
YOUTUBE_API_KEY = ""
HIGH_SIMILARITY = 0.99
LOW_SIMILARITY = 0.95

### END USER CONFIGURATION ###

# Returns the link and title of the lastest video on the SBFP website.
def get_latest_video():
    r = requests.get("http://superbestfriendsplay.com/category/videos/", "lxml")
    soup = BeautifulSoup(r.content, "html.parser")
    latest_list_item = soup.find("li", {"class": "archiveitem"})
    link = latest_list_item.find("div", {"class": "archivetitle"}).find("a").get("href")
    title = latest_list_item.find("div", {"class": "archivetitle"}).find("a").text
    return link, title

# Compares two strings and returns a float from 0.0 to 1.0 depending on
# the similarity of the strings.
def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()

# Returns whether or not the youtube video is found and the comment is posted.
def postYoutubeComment(title_ws, submission_posted, similarity):
    service = build("youtube", "v3", developerKey=YOUTUBE_API_KEY)
    playlistItems = service.playlistItems()
    results = 50 # Maximum allowed number of requested videos
    request = playlistItems.list(
        part="snippet", playlistId="UU84X0epDRFdTrybxEX8ZWkA",
        maxResults=results)
    uploads = request.execute()
    for i in range(0, results):
        iterationTitle = uploads["items"][i]["snippet"]["title"]
        # Compare how similar the two titles are. If it's almost
        # certain that they are the same video, post the comment
        if similar(iterationTitle, title_ws) > similarity:
            print("Video title on youtube channel: " + iterationTitle)
            print("Posting youtube url comment")
            videoId = (uploads["items"][i]["snippet"]["resourceId"]["videoId"])
            videoURL = "https://www.youtube.com/watch?v=" + videoId
            submission_posted.add_comment(
                "Here is the video on Youtube! " + videoURL)
            print(
                "Submission and comment posted. Sleeping",
                SLEEP_TIME, "seconds.")
            return True
    return False

if __name__ == "__main__":
    r = oaux.login()
    subreddit = r.get_subreddit(SUBREDDIT)
    latest_link_posted = ""
    while True:
        try:
            link_ws, title_ws = get_latest_video() # Latest link and title from website
            print("Latest video on website: " + title_ws)
            if link_ws != latest_link_posted: # If link_ws not already posted
                latest_link = link_ws
                print("Posting latest video")
                submission_posted = subreddit.submit(title_ws, url = link_ws)
                # submission for testing purposes
                # r.get_submission(submission_id = '3h5hzr')
                if postYoutubeComment(
                        title_ws, submission_posted, HIGH_SIMILARITY) == False:
                    print(
                        "Submission posted but not comment.",
                        "Could not find appropriate youtube video.",
                        "Trying again with a lower similarity ratio.")
                    if postYoutubeComment(
                            title_ws, submission_posted,
                            LOW_SIMILARITY) == False:
                        print(
                            "Still couldn't find youtube video,",
                            "please find it and comment manually.",
                            "Sleeping", SLEEP_TIME, "seconds")
            else:
                print(
                    "Not posting because I already posted. Sleeping",
                    SLEEP_TIME, seconds)
            time.sleep(SLEEP_TIME)
        except KeyboardInterrupt:
            print("Shutting down.")
            break
        except Exception as e:
            print("Something bad happened!", e)
            traceback.print_exc()
            print("Waiting", SLEEP_TIME, "seconds.")
            time.sleep(SLEEP_TIME)
<|endoftext|>


<commit_before>"Main module for running pychecker a Tkinter GUI for all the options"

import sys

from OptionTypes import *
from string import capitalize, strip

import Config

MAX_SUBBOX_ROWS = 7
MAX_BOX_COLS = 3

class ConfigDialog:
    "Dialog for editiong options"

    def __init__(self, tk):
        self._tk = tk
        self._cfg, _, _ = Config.setupFromArgs(sys.argv)

        self._optMap = {}
        self._opts = []
        for name, group in Config._OPTIONS:
          opts = []
          for _, useValue, longArg, member, description in group:
              value = None
              if member:
                  value = getattr(self._cfg, member)
                  description = member + ": " + capitalize(description)
                  description = strip(description)
              tk.option_add('*' + longArg + ".help", description)
              if useValue:
                  if type(value) == type([]):
                      field = List(longArg, value)
                  elif type(value) == type(1):
                      field = Number(longArg, int(value))
                  elif type(value) == type(''):
                      field = Text(longArg, value)
                  else:
                      field = Boolean(longArg, value)
              else:
                  field = Boolean(longArg, value)
              self._optMap[longArg] = field
              opts.append(field)
          self._opts.append( (name, opts))
        self._help = None

    def help(self, w):
        if self._help != w:
            help = w.option_get("help", "help")
            self._help.configure(text=help)

    def focus(self, ev):
        self.help(ev.widget)

    def click(self, ev):
        self.help(ev.widget)

    def add_fields(self, w, opts):
        count = 0
        for opt in opts:
            f = opt.field(w)
            c, r = divmod(count, MAX_SUBBOX_ROWS)
            f.grid(row=r, col=c, sticky=Tkinter.NSEW)
            count = count + 1

    def add_group(self, w, name, opts):
        colFrame = Tkinter.Frame(w)

        label = Tkinter.Label(colFrame, text=name + ":")
        label.grid(row=0, col=0, sticky=Tkinter.NSEW)

        gframe = Tkinter.Frame(colFrame, relief=Tkinter.GROOVE, borderwidth=2)
        gframe.grid(row=1, col=0, sticky=Tkinter.NSEW)
        self.add_fields(gframe, opts)

        label = Tkinter.Label(colFrame)
        label.grid(row=2, col=0, sticky=Tkinter.NSEW)
        colFrame.rowconfigure(2, weight=1)
        return colFrame

    def main(self):
        frame = Tkinter.Frame(self._tk, name="opts")
        frame.grid()
        tk.option_readfile('Options.ad')
        self._fields = {}
        row, col = 0, 0
        rowFrame = Tkinter.Frame(frame)
        rowFrame.grid(row=row)
        row = row + 1
        for name, opts in self._opts:
            w = self.add_group(rowFrame, name, opts)
            w.grid(row=row, col=col, sticky=Tkinter.NSEW)
            col = col + 1
            if col >= MAX_BOX_COLS:
                rowFrame=Tkinter.Frame(frame)
                rowFrame.grid(row=row, sticky=Tkinter.NSEW)
                col = 0
                row = row + 1

        self._help = Tkinter.Label(tk, name="helpBox")
        self._help.grid(row=row)
        self._help.config(takefocus=0)
        buttons = Tkinter.Frame(tk, name="buttons")
        ok = Tkinter.Button(buttons, name="ok", command=self.ok)
        ok.grid(row=row, col=0)
        default = Tkinter.Button(buttons, name="default", command=self.default)
        default.grid(row=row, col=1)
        cancel = Tkinter.Button(buttons, name="cancel", command=self.cancel)
        cancel.grid(row=row, col=2)
        buttons.grid()

        tk.bind_all('<FocusIn>', self.focus)
        tk.bind_all('<Enter>', self.focus)
        tk.bind_all('<ButtonPress>', self.click)
        tk.mainloop()

    def ok(self):
        opts = []
        # Pull command-line args
        for _, group in self._opts:
            for opt in group:
                arg = opt.arg()
                if arg:
                    opts.append(arg)
        print 'opts', opts

        # Calculate config
        self._cfg, _, _ = Config.setupFromArgs(opts)

        # Set controls based on new config
        for name, group in Config._OPTIONS:
            for _, _, longArg, member, _ in group:
                if member:
                    self._optMap[longArg].set(getattr(self._cfg, member))

    def default(self):
        self._cfg, _, _ = Config.setupFromArgs(sys.argv)
        for _, group in Config._OPTIONS:
            for _, _, longArg, member, _ in group:
                if member:
                    self._optMap[longArg].set(getattr(self._cfg, member))
                else:
                    self._optMap[longArg].set(0)

    def cancel(self):
        sys.exit(0)

if __name__=='__main__':
    tk = Tkinter.Tk()
    tk.title('PyChecker')
    ConfigDialog(tk).main()
<commit_msg>more layout... not quite so ugly<commit_after>"Main module for running pychecker a Tkinter GUI for all the options"

import sys

from OptionTypes import *
from string import capitalize, strip

import Config

MAX_SUBBOX_ROWS = 7
MAX_BOX_COLS = 3
PAD = 10

def col_weight(grid):
    unused, col = grid.grid_size()
    for c in range(col):
        grid.columnconfigure(c, weight=1)

class ConfigDialog:
    "Dialog for editiong options"

    def __init__(self, tk):
        self._tk = tk
        self._cfg, _, _ = Config.setupFromArgs(sys.argv)

        self._optMap = {}
        self._opts = []
        for name, group in Config._OPTIONS:
          opts = []
          for _, useValue, longArg, member, description in group:
              value = None
              if member:
                  value = getattr(self._cfg, member)
                  description = member + ": " + capitalize(description)
                  description = strip(description)
              tk.option_add('*' + longArg + ".help", description)
              if useValue:
                  if type(value) == type([]):
                      field = List(longArg, value)
                  elif type(value) == type(1):
                      field = Number(longArg, int(value))
                  elif type(value) == type(''):
                      field = Text(longArg, value)
                  else:
                      field = Boolean(longArg, value)
              else:
                  field = Boolean(longArg, value)
              self._optMap[longArg] = field
              opts.append(field)
          self._opts.append( (name, opts))
        self._help = None

    def help(self, w):
        if self._help != w:
            help = w.option_get("help", "help")
            self._help.configure(text=help)

    def focus(self, ev):
        self.help(ev.widget)

    def click(self, ev):
        self.help(ev.widget)

    def add_fields(self, w, opts):
        count = 0
        for opt in opts:
            f = opt.field(w)
            c, r = divmod(count, MAX_SUBBOX_ROWS)
            f.grid(row=r, col=c, sticky=Tkinter.NSEW)
            count = count + 1

    def add_group(self, w, name, opts):
        colFrame = Tkinter.Frame(w)

        label = Tkinter.Label(colFrame, text=name + ":")
        label.grid(row=0, col=0, sticky=Tkinter.NSEW)

        gframe = Tkinter.Frame(colFrame, relief=Tkinter.GROOVE, borderwidth=2)
        gframe.grid(row=1, col=0, sticky=Tkinter.NSEW)
        self.add_fields(gframe, opts)

        label = Tkinter.Label(colFrame)
        label.grid(row=2, col=0, sticky=Tkinter.NSEW)
        colFrame.rowconfigure(2, weight=1)
        return colFrame

    def main(self):
        frame = Tkinter.Frame(self._tk, name="opts")
        frame.grid()
        tk.option_readfile('Options.ad')
        self._fields = {}
        row, col = 0, 0
        rowFrame = Tkinter.Frame(frame)
        rowFrame.grid(row=row)
        row = row + 1
        for name, opts in self._opts:
            w = self.add_group(rowFrame, name, opts)
            w.grid(row=row, col=col, sticky=Tkinter.NSEW, padx=PAD)
            col = col + 1
            if col >= MAX_BOX_COLS:
                col_weight(rowFrame)
                rowFrame=Tkinter.Frame(frame)
                rowFrame.grid(row=row, sticky=Tkinter.NSEW)
                col = 0
                row = row + 1
        col_weight(rowFrame)

        self._help = Tkinter.Label(tk, name="helpBox")
        self._help.grid(row=row)
        self._help.config(takefocus=0)
        buttons = Tkinter.Frame(tk, name="buttons")
        ok = Tkinter.Button(buttons, name="ok", command=self.ok)
        ok.grid(row=row, col=0)
        default = Tkinter.Button(buttons, name="default", command=self.default)
        default.grid(row=row, col=1)
        cancel = Tkinter.Button(buttons, name="cancel", command=self.cancel)
        cancel.grid(row=row, col=2)
        buttons.grid()

        tk.bind_all('<FocusIn>', self.focus)
        tk.bind_all('<Enter>', self.focus)
        tk.bind_all('<ButtonPress>', self.click)
        tk.mainloop()

    def ok(self):
        opts = []
        # Pull command-line args
        for _, group in self._opts:
            for opt in group:
                arg = opt.arg()
                if arg:
                    opts.append(arg)
        print 'opts', opts

        # Calculate config
        self._cfg, _, _ = Config.setupFromArgs(opts)

        # Set controls based on new config
        for _, group in Config._OPTIONS:
            for _, _, longArg, member, _ in group:
                if member:
                    self._optMap[longArg].set(getattr(self._cfg, member))

    def default(self):
        self._cfg, _, _ = Config.setupFromArgs(sys.argv)
        for _, group in Config._OPTIONS:
            for _, _, longArg, member, _ in group:
                if member:
                    self._optMap[longArg].set(getattr(self._cfg, member))
                else:
                    self._optMap[longArg].set(0)

    def cancel(self):
        sys.exit(0)

if __name__=='__main__':
    tk = Tkinter.Tk()
    tk.title('PyChecker')
    ConfigDialog(tk).main()
<|endoftext|>


<commit_before>#!/usr/bin/python
"""videothread.py: Threaded, stopable video player class
Author: Wes Modes (wmodes@gmail.com)
Copyright: 2017, MIT """

# -*- coding: iso-8859-15 -*-
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import threading
import subprocess
import sys
import signal
import os
from time import sleep, time

# local imports
import ffprobe
from common import *
import config

nullin = open(os.devnull, 'r')
nullout = open(os.devnull, 'w')

#
# globals
#

omx_layer_content = 1
omx_layer_loop = 3
omx_layer_transition = 5
omx_player = 1

class VideoThread(threading.Thread):
    """Thread class with a stop() method. The thread itself checks
    regularly for the stopped() condition."""
    _example = """
        The class takes a a dictionary containing
        data about the videos to be played:
            {'name': "loop-idle1",         # if omitted, uses 'file'
                'file': "loop-idle1.mp4",   # looks in 'mediabase' for media
                'tags': 'loop',             # loop, transition, content, etc
                'start': 0.0,               # if omitted, assumes 0
                'length': 0.0,              # if omitted, assumes duration(filename)
                'disabled': True,           # if omitted, assumes False
             }
        """

    def __init__(self, video=None, media_dir=".", debug=0):
        super(VideoThread, self).__init__()
        self._stop = threading.Event()
        # passed parameters
        self.video = video
        self.media_dir = media_dir
        # internal flags and vars
        self._debug_flag = debug
        self._last_debug_caller = None
        self._current_video = None
        self._player_pgid = None
        self._end_time = 0

    def set_sequence(self, video=None):
        self.video = video

    def stop(self):
        self._debug("Stop flag set")
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def debug(self, debug_flag):
        self._debug_flag = debug_flag

    def _debug(self, *args, **kwargs):
        """Produce debug message, indenting if from same calling function"""
        level = 1
        if ("level" in kwargs):
            level = kwargs["level"]
        elif("l" in kwargs):
            level = kwargs["l"]
        if (self._debug_flag >= level):
            text = " ".join(list(map(str, args)))
            # get name of calling function
            caller = sys._getframe(1).f_code.co_name
            if (caller == self._last_debug_caller):
                #print "  debug: %s: %s" % (caller, text)
                logger.info("  debug: %s: %s" % (caller, text))
            else:
                #print "debug: %s: %s" % (caller, text)
                logger.info("debug: %s: %s" % (caller, text))
            # save last calling function
            self._last_debug_caller = caller

    def run(self):
        if not isinstance(self.video, dict):
            raise ValueError(self._example)
        if not self.stopped():
            self._start_video(self.video)

    def _start_video(self, video):
        """Starts a video. Takes a video object """
        # add media_dir to filename
        global omx_layer_content, omx_layer_loop, omx_layer_transition, omx_player
        filename = self.media_dir + '/' + video['file']
        # check to make sure we've passed the right thing
        if not isinstance(video, dict):
            raise ValueError(self._example)
        # set video name if we have it
        if 'name' in video:
            name = video['name']
        else:
            name = video['file']
        # skip this video if disabled in db
        if 'disabled' in video and video['disabled']:
            self._debug("Not played:", name, "disabled")
            return
        #debug messages
        self._debug("Starting %s in %s" % (name, self.media_dir))
        self._debug("Video data:", video)
        # get length
        filelength = self._get_length(filename)
        if ('length' not in video or video['length'] == 0.0):
            length = filelength
        else:
            length = video['length']
        # get start
        if 'start' in video:
            start = video['start']
        else:
            start = 0.0
        # if start is too large, set it to 0
        if (start >= filelength):
            start = 0.0
        # if length is too large, scale it back, unless loop
        if (('loop' not in video['tags']) and (start + length >= filelength)):
            length = filelength - start
        # store this for later
        self._current_video = video
        # debugging output
        self._debug("name: %s (%s)" % (name, filename))
        self._debug("tags: %s" % video['tags'])
        self._debug("start: %.1fs, end: %.1fs, len: %.1fs" %
                    (start, start+length, length))
        # each time we switch to a new video, we switch the layer
        # this will effectively toggle the 3 layer variables between (1,2), (3,4), and (5,6)
        omx_layer_content = 1 if (omx_layer_content != 1) else 2
        omx_layer_loop = 3 if (omx_layer_loop != 3) else 4
        omx_layer_transition = 5 if (omx_layer_transition != 5) else 6
        # we also toggle the virtual player
        omx_player = 1 if (omx_player != 1) else 2
        # build omxplayer command
        if ('loop' in video['tags']):
            my_cmd = " ".join(config.loop_cmd) \
                    + " '" + filename + "'" \
                    % (omx_layer_loop, omx_player)
        elif ('transition' in video['tags']):
            my_cmd = " ".join(config.transition_cmd + ['--pos', str(start)]) \
                    + " '" + filename + "'" \
                    % (omx_layer_transition, omx_player)
        else:
            print " ".join(config.content_cmd + ['--pos', str(start)]) \
                    + " '" + filename + "'"
            my_cmd = " ".join(config.content_cmd + ['--pos', str(start)]) \
                    + " '" + filename + "'" \
                    % (omx_layer_content, omx_player)
        self._debug("cmd:", my_cmd, l=2)
        # launch the player, saving the process handle
        # TODO: after debugging, replace 'if True' with 'try' and enable 'except'
        #if True:
        try:
            process = None
            process = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin,
                                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
           # save this process group id
            pgid = os.getpgid(process.pid)
            self._player_pgid = pgid
            self._debug("Starting process: %i (%s)" % (pgid, name))
            # If we have a loop
            if ('loop' in video['tags']):
                self._debug("Looping %.2fs for %s (pid %i)" %
                            (length - config.inter_video_delay, name, pgid))
            # otherwise
            else:
                self._debug("Waiting %.2fs for %s (pid %i)" %
                            (length - config.inter_video_delay, name, pgid))
            # generate expected _end_time (now + length)
            start_time = time()
            self._end_time = start_time + length
            # wait until the end times (but also allowing that the stop flag is triggered)
            while ((time() <= self._end_time) and
                    not self.stopped()):
                sleep(0.1)
            # emerging from this loop, either
            #   1) player ended gracefully
            #   2) the video is looped, and so needs to be stopped
            #   3) the thread received a stop order, and so video needs to be killed
            #   4) something else, like player hung
            # In any case, we kill the process group of the video, if we can
            if process.poll() is None:
                self._stop_video(pgid, name)
            # Now that we have ended one way or the other, we should be able to get stdout/stderr
            # Report if we had any problems starting omxplayer
            stdoutdata, stderrdata = process.communicate()
            returncode = process.returncode
            # if the process failed, let's log the output
            if (returncode != 0):
                self._debug("Error starting omxplayer for %s\n%s\n%s" % \
                            (name, str(stdoutdata), str(stderrdata)))
        except Exception as e:
             self._debug("Error starting omxplayer for %s\n%s" % (name, str(e)))

    def wait_for_end(self):
        """Wait for end of video in tight loop. This provides a synchronous mechanism to wait
        for the end of a video."""
        #
        # The run() method is called asynchronously, so it is possible to call this
        # method before run() has recorded the expected _end_time.
        # thus we wait until the end times (but also allowing that the stop flag is triggered)
        while (not self._end_time and
                not self.stopped()):
            sleep(0.1)
        #self._debug("Waiting for end of video")
        # now wait until time expires (or we are interrupted because stop flag is triggered)
        while ((time() <= self._end_time) and
                not self.stopped()):
            sleep(0.1)

    def _stop_video(self, pgid, name):
        self._debug("Sending SIGTERM to process %i (%s)" % (pgid, name))
        try:
            os.killpg(pgid, signal.SIGTERM)
            self._player_pgid = None
            self._current_video = None
        except OSError, e:
            self._debug("Couldn't terminate %s (pid %i)\n%s" % (name, pgid, str(e)))
            pass

    def _get_length(self, filename):
        self._debug("Getting duration of %s" % filename)
        length = ffprobe.duration(filename)
        if length is None:
            length = 0
        return length

def main():
    media_dir = "media"
    films = [
          { "file" : "launch_party_timelapse.mp4",
            "tags" : [ "footage" ],
            "length" : 20,
          },
          { "file" : "secret_history_dorris_turner_what_does_the_river_mean_to_you.mp4",
            "tags" : [ "interview" ],
            "length" : 20,
          },
          { "file" : "la_crosse_boathouses_loop.mp4",
            "tags" : [ "title" ],
            "length" : 200,
          },
        ]
    for film in films:
        print "Starting threaded sequence"
        video = VideoThread(film, media_dir, debug=2)
        video.start()
        print "Sequence started"
        video.wait_for_end()

if __name__ == "__main__":
    main()
<commit_msg>debug<commit_after>#!/usr/bin/python
"""videothread.py: Threaded, stopable video player class
Author: Wes Modes (wmodes@gmail.com)
Copyright: 2017, MIT """

# -*- coding: iso-8859-15 -*-
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import threading
import subprocess
import sys
import signal
import os
from time import sleep, time

# local imports
import ffprobe
from common import *
import config

nullin = open(os.devnull, 'r')
nullout = open(os.devnull, 'w')

#
# globals
#

omx_layer_content = 1
omx_layer_loop = 3
omx_layer_transition = 5
omx_player = 1

class VideoThread(threading.Thread):
    """Thread class with a stop() method. The thread itself checks
    regularly for the stopped() condition."""
    _example = """
        The class takes a a dictionary containing
        data about the videos to be played:
            {'name': "loop-idle1",         # if omitted, uses 'file'
                'file': "loop-idle1.mp4",   # looks in 'mediabase' for media
                'tags': 'loop',             # loop, transition, content, etc
                'start': 0.0,               # if omitted, assumes 0
                'length': 0.0,              # if omitted, assumes duration(filename)
                'disabled': True,           # if omitted, assumes False
             }
        """

    def __init__(self, video=None, media_dir=".", debug=0):
        super(VideoThread, self).__init__()
        self._stop = threading.Event()
        # passed parameters
        self.video = video
        self.media_dir = media_dir
        # internal flags and vars
        self._debug_flag = debug
        self._last_debug_caller = None
        self._current_video = None
        self._player_pgid = None
        self._end_time = 0

    def set_sequence(self, video=None):
        self.video = video

    def stop(self):
        self._debug("Stop flag set")
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def debug(self, debug_flag):
        self._debug_flag = debug_flag

    def _debug(self, *args, **kwargs):
        """Produce debug message, indenting if from same calling function"""
        level = 1
        if ("level" in kwargs):
            level = kwargs["level"]
        elif("l" in kwargs):
            level = kwargs["l"]
        if (self._debug_flag >= level):
            text = " ".join(list(map(str, args)))
            # get name of calling function
            caller = sys._getframe(1).f_code.co_name
            if (caller == self._last_debug_caller):
                #print "  debug: %s: %s" % (caller, text)
                logger.info("  debug: %s: %s" % (caller, text))
            else:
                #print "debug: %s: %s" % (caller, text)
                logger.info("debug: %s: %s" % (caller, text))
            # save last calling function
            self._last_debug_caller = caller

    def run(self):
        if not isinstance(self.video, dict):
            raise ValueError(self._example)
        if not self.stopped():
            self._start_video(self.video)

    def _start_video(self, video):
        """Starts a video. Takes a video object """
        # add media_dir to filename
        global omx_layer_content, omx_layer_loop, omx_layer_transition, omx_player
        filename = self.media_dir + '/' + video['file']
        # check to make sure we've passed the right thing
        if not isinstance(video, dict):
            raise ValueError(self._example)
        # set video name if we have it
        if 'name' in video:
            name = video['name']
        else:
            name = video['file']
        # skip this video if disabled in db
        if 'disabled' in video and video['disabled']:
            self._debug("Not played:", name, "disabled")
            return
        #debug messages
        self._debug("Starting %s in %s" % (name, self.media_dir))
        self._debug("Video data:", video)
        # get length
        filelength = self._get_length(filename)
        if ('length' not in video or video['length'] == 0.0):
            length = filelength
        else:
            length = video['length']
        # get start
        if 'start' in video:
            start = video['start']
        else:
            start = 0.0
        # if start is too large, set it to 0
        if (start >= filelength):
            start = 0.0
        # if length is too large, scale it back, unless loop
        if (('loop' not in video['tags']) and (start + length >= filelength)):
            length = filelength - start
        # store this for later
        self._current_video = video
        # debugging output
        self._debug("name: %s (%s)" % (name, filename))
        self._debug("tags: %s" % video['tags'])
        self._debug("start: %.1fs, end: %.1fs, len: %.1fs" %
                    (start, start+length, length))
        # each time we switch to a new video, we switch the layer
        # this will effectively toggle the 3 layer variables between (1,2), (3,4), and (5,6)
        omx_layer_content = 1 if (omx_layer_content != 1) else 2
        omx_layer_loop = 3 if (omx_layer_loop != 3) else 4
        omx_layer_transition = 5 if (omx_layer_transition != 5) else 6
        # we also toggle the virtual player
        omx_player = 1 if (omx_player != 1) else 2
        # build omxplayer command
        if ('loop' in video['tags']):
            my_cmd = " ".join(config.loop_cmd) % \
                    (omx_layer_content, omx_player)
            my_cmd += " '" + filename + "'"
        elif ('transition' in video['tags']):
            my_cmd = " ".join(config.transition_cmd + ['--pos', str(start)]) % \
                    (omx_layer_content, omx_player)
            my_cmd += " '" + filename + "'"
        else:
            my_cmd = " ".join(config.content_cmd + ['--pos', str(start)]) % \
                    (omx_layer_content, omx_player)
            my_cmd += " '" + filename + "'"
            print my_cmd
        self._debug("cmd:", my_cmd, l=2)
        # launch the player, saving the process handle
        # TODO: after debugging, replace 'if True' with 'try' and enable 'except'
        #if True:
        try:
            process = None
            process = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin,
                                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
           # save this process group id
            pgid = os.getpgid(process.pid)
            self._player_pgid = pgid
            self._debug("Starting process: %i (%s)" % (pgid, name))
            # If we have a loop
            if ('loop' in video['tags']):
                self._debug("Looping %.2fs for %s (pid %i)" %
                            (length - config.inter_video_delay, name, pgid))
            # otherwise
            else:
                self._debug("Waiting %.2fs for %s (pid %i)" %
                            (length - config.inter_video_delay, name, pgid))
            # generate expected _end_time (now + length)
            start_time = time()
            self._end_time = start_time + length
            # wait until the end times (but also allowing that the stop flag is triggered)
            while ((time() <= self._end_time) and
                    not self.stopped()):
                sleep(0.1)
            # emerging from this loop, either
            #   1) player ended gracefully
            #   2) the video is looped, and so needs to be stopped
            #   3) the thread received a stop order, and so video needs to be killed
            #   4) something else, like player hung
            # In any case, we kill the process group of the video, if we can
            if process.poll() is None:
                self._stop_video(pgid, name)
            # Now that we have ended one way or the other, we should be able to get stdout/stderr
            # Report if we had any problems starting omxplayer
            stdoutdata, stderrdata = process.communicate()
            returncode = process.returncode
            # if the process failed, let's log the output
            if (returncode != 0):
                self._debug("Error starting omxplayer for %s\n%s\n%s" % \
                            (name, str(stdoutdata), str(stderrdata)))
        except Exception as e:
             self._debug("Error starting omxplayer for %s\n%s" % (name, str(e)))

    def wait_for_end(self):
        """Wait for end of video in tight loop. This provides a synchronous mechanism to wait
        for the end of a video."""
        #
        # The run() method is called asynchronously, so it is possible to call this
        # method before run() has recorded the expected _end_time.
        # thus we wait until the end times (but also allowing that the stop flag is triggered)
        while (not self._end_time and
                not self.stopped()):
            sleep(0.1)
        #self._debug("Waiting for end of video")
        # now wait until time expires (or we are interrupted because stop flag is triggered)
        while ((time() <= self._end_time) and
                not self.stopped()):
            sleep(0.1)

    def _stop_video(self, pgid, name):
        self._debug("Sending SIGTERM to process %i (%s)" % (pgid, name))
        try:
            os.killpg(pgid, signal.SIGTERM)
            self._player_pgid = None
            self._current_video = None
        except OSError, e:
            self._debug("Couldn't terminate %s (pid %i)\n%s" % (name, pgid, str(e)))
            pass

    def _get_length(self, filename):
        self._debug("Getting duration of %s" % filename)
        length = ffprobe.duration(filename)
        if length is None:
            length = 0
        return length

def main():
    media_dir = "media"
    films = [
          { "file" : "launch_party_timelapse.mp4",
            "tags" : [ "footage" ],
            "length" : 20,
          },
          { "file" : "secret_history_dorris_turner_what_does_the_river_mean_to_you.mp4",
            "tags" : [ "interview" ],
            "length" : 20,
          },
          { "file" : "la_crosse_boathouses_loop.mp4",
            "tags" : [ "title" ],
            "length" : 200,
          },
        ]
    for film in films:
        print "Starting threaded sequence"
        video = VideoThread(film, media_dir, debug=2)
        video.start()
        print "Sequence started"
        video.wait_for_end()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2001, MetaSlash Inc.  All rights reserved.

"Helper functions for printing out info about objects"

def printFunction(spaces, prefix, func, className = None) :
    params = ''
    argcount = func.func_code.co_argcount
    defaultArgStart = argcount
    if func.func_defaults != None :
        defaultArgStart = argcount - len(func.func_defaults)
    for i in range(0, argcount) :
        arg = func.func_code.co_varnames[i]
        if i >= defaultArgStart :
            arg = arg + " = %s" % func.func_defaults[i - defaultArgStart]
        params = params + "%s, " % arg
    params = "(%s)" % params[:-2]
    if className == None :
        className = ""
    else :
        className = className + "."
    print "%s%s%s%s%s" % (spaces, prefix, className, func.func_name, params)

def module(module) :
    print "Module: ", module.moduleName
    if module.module == None :
        return

    print "  Imports:  ", module.modules.keys()
    print "  Variables:", module.variables.keys()
    print ""
    for function in module.functions.values() :
        printFunction("  ", "Function:  ", function.function)
    print ""
    for c in module.classes.values() :
        for method in c.methods.values() :
            printFunction("  ", "", method, c.name)
        print ""

def attrs(object) :
    for attr in dir(object) :
        print " %s: %s" % (attr, `getattr(object, attr)`)
<commit_msg>get to work w/new structure<commit_after>#!/usr/bin/env python

# Copyright (c) 2001, MetaSlash Inc.  All rights reserved.

"Helper functions for printing out info about objects"

def printFunction(spaces, prefix, func, className = None) :
    params = ''
    argcount = func.func_code.co_argcount
    defaultArgStart = argcount
    if func.func_defaults != None :
        defaultArgStart = argcount - len(func.func_defaults)
    for i in range(0, argcount) :
        arg = func.func_code.co_varnames[i]
        if i >= defaultArgStart :
            arg = arg + " = %s" % func.func_defaults[i - defaultArgStart]
        params = params + "%s, " % arg
    params = "(%s)" % params[:-2]
    if className == None :
        className = ""
    else :
        className = className + "."
    print "%s%s%s%s%s" % (spaces, prefix, className, func.func_name, params)

def module(module) :
    print "Module: ", module.moduleName
    if module.module == None :
        return

    print "  Imports:  ", module.modules.keys()
    print "  Variables:", module.variables.keys()
    print ""
    for function in module.functions.values() :
        printFunction("  ", "Function:  ", function.function)
    print ""
    for c in module.classes.values() :
        for method in c.methods.values() :
            if method != None :
                printFunction("  ", "", method.function, c.name)
        print ""

def attrs(object) :
    for attr in dir(object) :
        print " %s: %s" % (attr, `getattr(object, attr)`)
<|endoftext|>


<commit_before># encoding=UTF-8

# Copyright © 2014-2017 Jakub Wilk <jwilk@jwilk.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
pydiatra re checks
'''

import ast
import inspect
import operator
import re
import sre_parse
import sys
import warnings

from . import tags

tag = tags.Tag

def analyze_re_functions():
    funcs = [
        re.compile,
        re.findall,
        re.finditer,
        re.match,
        re.search,
        re.split,
        re.sub,
        re.subn,
        re.template,
    ]
    if sys.version_info >= (3, 4):
        funcs += [
            re.fullmatch
        ]
    data = {}
    for func in funcs:
        if sys.version_info < (3, 0):
            func_args = inspect.getargspec(func).args
        elif sys.version_info < (3, 3):
            func_args = inspect.getfullargspec(func).args
        else:
            func_args = [
                arg.name
                for arg in inspect.signature(func).parameters.values()
                if arg.kind in (arg.POSITIONAL_ONLY, arg.POSITIONAL_OR_KEYWORD)
            ]
        assert 'pattern' in func_args
        if sys.version_info >= (2, 7):
            assert 'flags' in func_args
        data[func.__name__] = func_args
    return data

re_functions = analyze_re_functions()

# pylint: disable=redefined-builtin
if sys.version_info >= (3,):
    long = int
    unichr = chr
    unicode = str
else:
    ascii = repr
# pylint: enable=redefined-builtin

def format_char_range(rng, tp):
    if tp in (str, bytes):
        def fmt(i):
            return ascii(chr(i))[1:-1]
    else:
        def fmt(i):
            return repr(unichr(i))[2:-1]
    x, y = map(fmt, rng)
    if x != y:
        return x + '-' + y
    else:
        return x

class ReVisitor(object):

    def __init__(self, tp, path, lineno):
        self.tp = tp
        self.path = path
        self.lineno = lineno

    def tag(self, lineno, *args):
        assert lineno == self.lineno
        return tag(self.path, lineno, *args)

    def _normalize_op(self, op):
        if sys.version_info >= (3, 5):
            op = repr(op).lower()
        if type(op) != str:  # pylint: disable=unidiomatic-typecheck
            raise TypeError
        return op

    def visit(self, node):
        if not isinstance(node, sre_parse.SubPattern):
            raise TypeError('{0!r} is not a subpattern'.format(node))
        for op, args in node.data:
            if not isinstance(args, (list, tuple)):
                args = (args,)
            op = self._normalize_op(op)
            method = 'visit_' + op
            visitor = getattr(self, method, self.generic_visit)
            for t in visitor(*args):
                yield t

    def generic_visit(self, *args):
        for arg in args:
            if isinstance(arg, (list, tuple)):
                for t in self.generic_visit(*arg):
                    yield t
            elif isinstance(arg, sre_parse.SubPattern):
                for t in self.visit(arg):
                    yield t
            elif isinstance(arg, (int, long, str)):
                pass
            elif arg is None:
                pass
            else:
                raise TypeError('{0!r} has unexpected type'.format(arg))

    def visit_in(self, *args):
        ranges = []
        for op, arg in args:
            op = self._normalize_op(op)
            if op == 'range':
                ranges += [arg]
            elif op == 'literal':
                ranges += [(arg, arg)]
        seen_duplicate_range = False
        seen_overlapping_ranges = False
        if len(ranges) >= 2:
            ranges.sort()
            for i in range(len(ranges) - 1):
                r1 = ranges[i]
                r2 = ranges[i + 1]
                if r1 == r2:
                    if not seen_duplicate_range:
                        yield self.tag(self.lineno, 'regexp-duplicate-range',
                            format_char_range(r1, tp=self.tp),
                        )
                    seen_duplicate_range = True
                elif r1[1] >= r2[0]:
                    if not seen_overlapping_ranges:
                        yield self.tag(self.lineno, 'regexp-overlapping-ranges',
                            format_char_range(r1, tp=self.tp),
                            format_char_range(r2, tp=self.tp),
                        )
                    seen_overlapping_ranges = True
        for t in self.generic_visit(*args):
            yield t

class BadConst(Exception):
    pass

class Evaluator(ast.NodeVisitor):

    def generic_visit(self, node):
        raise BadConst

    def visit_Str(self, node):
        return node.s

    def visit_Bytes(self, node):
        return node.s

    def visit_Attribute(self, node):
        value = node.value
        if isinstance(value, ast.Name) and value.id == 're':
            value = getattr(re, node.attr, None)
            if isinstance(value, int):
                return value
        raise BadConst

    def visit_BinOp(self, node):
        if isinstance(node.op, ast.BitOr):
            op = operator.or_
        elif isinstance(node.op, ast.Add):
            op = operator.add
        else:
            raise BadConst
        values = [
            self.visit(node.left),
            self.visit(node.right),
        ]
        if all(isinstance(v, int) for v in values):
            return op(*values)
        else:
            raise BadConst

evaluator = Evaluator()

def eval_const(node):
    try:
        return evaluator.visit(node)
    except BadConst as exc:
        return exc

def check(owner, node):
    if sys.version_info < (3, 5):
        if node.starargs:
            return
        if node.kwargs:
            return
        starred_tp = ()
    else:
        starred_tp = ast.Starred
    func_name = node.func.attr
    try:
        argnames = re_functions[func_name]
    except KeyError:
        return
    args = {}
    for argname, argnode in zip(argnames, node.args):
        if isinstance(argnode, starred_tp):
            return
        args[argname] = eval_const(argnode)
    for argnode in node.keywords:
        if argnode.arg is None:
            return
        args[argnode.arg] = eval_const(argnode.value)
    pattern = args.get('pattern')
    if not isinstance(pattern, (unicode, str, bytes)):
        return
    flags = args.get('flags', 0)
    if not isinstance(flags, int):
        return
    if func_name == 'template':
        flags |= re.TEMPLATE
    flags &= ~re.DEBUG
    try:
        with warnings.catch_warnings(record=True) as wrns:
            warnings.simplefilter('default')
            subpattern = sre_parse.parse(pattern, flags=flags)
    except Exception as exc:  # pylint: disable=broad-except
        yield owner.tag(node.lineno, 'regexp-syntax-error', str(exc))
    else:
        for wrn in wrns:
            yield owner.tag(node.lineno, 'regexp-syntax-warning', str(wrn.message))
        try:
            re.compile(pattern, flags=flags)
        except Exception as exc:  # pylint: disable=broad-except
            yield owner.tag(node.lineno, 'regexp-syntax-error', str(exc))
        else:
            re_visitor = ReVisitor(tp=type(pattern), path=owner.path, lineno=node.lineno)
            for t in re_visitor.visit(subpattern):
                yield t

__all__ = ['check']

# vim:ts=4 sts=4 sw=4 et
<commit_msg>pydiatra/check_re: silence various Pylint warnings.<commit_after># encoding=UTF-8

# Copyright © 2014-2017 Jakub Wilk <jwilk@jwilk.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
pydiatra re checks
'''

import ast
import inspect
import operator
import re
import sre_parse
import sys
import warnings

from . import tags

tag = tags.Tag

def analyze_re_functions():
    funcs = [
        re.compile,
        re.findall,
        re.finditer,
        re.match,
        re.search,
        re.split,
        re.sub,
        re.subn,
        re.template,
    ]
    if sys.version_info >= (3, 4):
        funcs += [
            re.fullmatch  # pylint: disable=no-member
        ]
    data = {}
    for func in funcs:
        # pylint: disable=deprecated-method,no-member
        if sys.version_info < (3, 0):
            func_args = inspect.getargspec(func).args
        elif sys.version_info < (3, 3):
            func_args = inspect.getfullargspec(func).args
        else:
            func_args = [
                arg.name
                for arg in inspect.signature(func).parameters.values()
                if arg.kind in (arg.POSITIONAL_ONLY, arg.POSITIONAL_OR_KEYWORD)
            ]
        # pylint: enable=deprecated-method,no-member
        assert 'pattern' in func_args
        if sys.version_info >= (2, 7):
            assert 'flags' in func_args
        data[func.__name__] = func_args
    return data

re_functions = analyze_re_functions()

# pylint: disable=redefined-builtin
if sys.version_info >= (3,):
    long = int
    unichr = chr
    unicode = str
else:
    ascii = repr
# pylint: enable=redefined-builtin

def format_char_range(rng, tp):
    if tp in (str, bytes):
        def fmt(i):
            return ascii(chr(i))[1:-1]
    else:
        def fmt(i):
            return repr(unichr(i))[2:-1]
    x, y = map(fmt, rng)
    if x != y:
        return x + '-' + y
    else:
        return x

class ReVisitor(object):

    def __init__(self, tp, path, lineno):
        self.tp = tp
        self.path = path
        self.lineno = lineno

    def tag(self, lineno, *args):
        assert lineno == self.lineno
        return tag(self.path, lineno, *args)

    def _normalize_op(self, op):
        if sys.version_info >= (3, 5):
            op = repr(op).lower()
        if type(op) != str:  # pylint: disable=unidiomatic-typecheck
            raise TypeError
        return op

    def visit(self, node):
        if not isinstance(node, sre_parse.SubPattern):
            raise TypeError('{0!r} is not a subpattern'.format(node))
        for op, args in node.data:
            if not isinstance(args, (list, tuple)):
                args = (args,)
            op = self._normalize_op(op)
            method = 'visit_' + op
            visitor = getattr(self, method, self.generic_visit)
            for t in visitor(*args):
                yield t

    def generic_visit(self, *args):
        for arg in args:
            if isinstance(arg, (list, tuple)):
                for t in self.generic_visit(*arg):
                    yield t
            elif isinstance(arg, sre_parse.SubPattern):
                for t in self.visit(arg):
                    yield t
            elif isinstance(arg, (int, long, str)):
                pass
            elif arg is None:
                pass
            else:
                raise TypeError('{0!r} has unexpected type'.format(arg))

    def visit_in(self, *args):
        ranges = []
        for op, arg in args:
            op = self._normalize_op(op)
            if op == 'range':
                ranges += [arg]
            elif op == 'literal':
                ranges += [(arg, arg)]
        seen_duplicate_range = False
        seen_overlapping_ranges = False
        if len(ranges) >= 2:
            ranges.sort()
            for i in range(len(ranges) - 1):
                r1 = ranges[i]
                r2 = ranges[i + 1]
                if r1 == r2:
                    if not seen_duplicate_range:
                        yield self.tag(self.lineno, 'regexp-duplicate-range',
                            format_char_range(r1, tp=self.tp),
                        )
                    seen_duplicate_range = True
                elif r1[1] >= r2[0]:
                    if not seen_overlapping_ranges:
                        yield self.tag(self.lineno, 'regexp-overlapping-ranges',
                            format_char_range(r1, tp=self.tp),
                            format_char_range(r2, tp=self.tp),
                        )
                    seen_overlapping_ranges = True
        for t in self.generic_visit(*args):
            yield t

class BadConst(Exception):
    pass

class Evaluator(ast.NodeVisitor):

    def generic_visit(self, node):
        raise BadConst

    def visit_Str(self, node):
        return node.s

    def visit_Bytes(self, node):
        return node.s

    def visit_Attribute(self, node):
        value = node.value
        if isinstance(value, ast.Name) and value.id == 're':
            value = getattr(re, node.attr, None)
            if isinstance(value, int):
                return value
        raise BadConst

    def visit_BinOp(self, node):
        if isinstance(node.op, ast.BitOr):
            op = operator.or_
        elif isinstance(node.op, ast.Add):
            op = operator.add
        else:
            raise BadConst
        values = [
            self.visit(node.left),
            self.visit(node.right),
        ]
        if all(isinstance(v, int) for v in values):
            return op(*values)
        else:
            raise BadConst

evaluator = Evaluator()

def eval_const(node):
    try:
        return evaluator.visit(node)
    except BadConst as exc:
        return exc

def check(owner, node):
    if sys.version_info < (3, 5):
        if node.starargs:
            return
        if node.kwargs:
            return
        starred_tp = ()
    else:
        starred_tp = ast.Starred  # pylint: disable=no-member
    func_name = node.func.attr
    try:
        argnames = re_functions[func_name]
    except KeyError:
        return
    args = {}
    for argname, argnode in zip(argnames, node.args):
        if isinstance(argnode, starred_tp):
            return
        args[argname] = eval_const(argnode)
    for argnode in node.keywords:
        if argnode.arg is None:
            return
        args[argnode.arg] = eval_const(argnode.value)
    pattern = args.get('pattern')
    if not isinstance(pattern, (unicode, str, bytes)):
        return
    flags = args.get('flags', 0)
    if not isinstance(flags, int):
        return
    if func_name == 'template':
        flags |= re.TEMPLATE
    flags &= ~re.DEBUG
    try:
        with warnings.catch_warnings(record=True) as wrns:
            warnings.simplefilter('default')
            subpattern = sre_parse.parse(pattern, flags=flags)
    except Exception as exc:  # pylint: disable=broad-except
        yield owner.tag(node.lineno, 'regexp-syntax-error', str(exc))
    else:
        for wrn in wrns:
            yield owner.tag(node.lineno, 'regexp-syntax-warning', str(wrn.message))
        try:
            re.compile(pattern, flags=flags)
        except Exception as exc:  # pylint: disable=broad-except
            yield owner.tag(node.lineno, 'regexp-syntax-error', str(exc))
        else:
            re_visitor = ReVisitor(tp=type(pattern), path=owner.path, lineno=node.lineno)
            for t in re_visitor.visit(subpattern):
                yield t

__all__ = ['check']

# vim:ts=4 sts=4 sw=4 et
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""pyeasyga

A simple and easy-to-use genetic algorithm implementation library in Python.

For a bit array solution representation, simply instantiate the
GeneticAlgorithm class with input data, define and supply a fitness function,
run the Genetic Algorithm, and retrieve the solution!

Other solution representations will require setting some more attributes.

"""

__author__ = 'Ayodeji Remi-Omosowon'
__email__ = 'remiomosowon@gmail.com'
__version__ = '0.2.0'
<commit_msg>Update to version v0.2.1<commit_after># -*- coding: utf-8 -*-
"""pyeasyga

A simple and easy-to-use genetic algorithm implementation library in Python.

For a bit array solution representation, simply instantiate the
GeneticAlgorithm class with input data, define and supply a fitness function,
run the Genetic Algorithm, and retrieve the solution!

Other solution representations will require setting some more attributes.

"""

__author__ = 'Ayodeji Remi-Omosowon'
__email__ = 'remiomosowon@gmail.com'
__version__ = '0.2.1'
<|endoftext|>


<commit_before>import json
import sys
from jsonpath_ng import jsonpath, parse
import re
from io import StringIO
from lxml import etree

# noinspection PyMethodMayBeStatic
class PythonAgentExecutor(object):

    def __get_accessed_method(self, key):
        accessed_resources_set.add(key)

    def __disable_standard_io(self):
        old_io = (sys.stdin, sys.stdout, sys.stderr, sys.exit)
        sys.stdin, sys.stdout, sys.stderr, sys.exit = (None, None, None, lambda *x, **y: None)
        return old_io

    def __enable_standard_io(self, old_io):
        (sys.stdin, sys.stdout, sys.stderr, sys.exit) = old_io

    def cs_regex(self, str, regex):
        x = re.search(regex, str)
        return None if x is None else x.group()

    def cs_xpath_query(self, str, xpath):
        f = StringIO(str)
        tree = etree.parse(f)
        r = tree.xpath(xpath)
        return r[0] if len(r) > 0 else None

    def cs_json_query(self, str, json_path):
        json_data = json.loads(str)
        jsonpath_expr = parse(json_path)
        x = jsonpath_expr.find(json_data)
        return x[0].value if len(x) > 0 else None

    def __init_context(self, payload):
        global sys_prop
        global get_sp
        global get
        global cs_append
        global cs_prepend
        global cs_replace
        global cs_round
        global cs_extract_number
        global cs_substring
        global cs_to_upper
        global cs_to_lower
        global accessed
        global accessed_resources_set
        accessed_resources_set = set()
        context = payload["context"]

        if "sys_prop" in context:
            sys_prop = context["sys_prop"]
            accessed = self.__get_accessed_method

        env_setup = payload["envSetup"]
        get_sp = None
        get = None
        cs_append = None
        cs_prepend = None
        cs_replace = None
        cs_round = None
        cs_extract_number = None
        cs_substring = None
        cs_to_upper = None
        cs_to_lower = None
        exec (env_setup, globals())

    def main(self):
        try:
            raw_inputs = input().encode(sys.stdin.encoding).decode()
            payload = json.loads(raw_inputs)
            expression = payload["expression"]
            context = payload["context"]
            self.__init_context(payload)

            smaller_context = AccessAwareDict({"get_sp": get_sp, "get": get,
                                               "cs_append": cs_append,
                                               "cs_prepend": cs_prepend,
                                               "cs_replace": cs_replace,
                                               "cs_round": cs_round,
                                               "cs_extract_number": cs_extract_number,
                                               "cs_substring": cs_substring,
                                               "cs_to_upper": cs_to_upper,
                                               "cs_to_lower": cs_to_lower,
                                               "cs_regex": self.cs_regex,
                                               "cs_xpath_query": self.cs_xpath_query,
                                               "cs_json_query": self.cs_json_query,
                                               })

            for x in dir(__builtins__):
                smaller_context[x] = eval(x)

            for key, var in context.items():
                smaller_context[key] = var

            old_io = self.__disable_standard_io()

            try:
                expr_result = eval(expression, smaller_context)
                return_type = type(expr_result).__name__
                if return_type not in ['str', 'int', 'bool']:
                    return_type = 'str'
                final_result = {"returnResult": expr_result,
                                "accessedResources": list(accessed_resources_set),
                                "returnType": return_type}
            finally:
                self.__enable_standard_io(old_io)
        except Exception as e:
            final_result = {"exception": str(e)}

        print(json.dumps(final_result))

class AccessAwareDict(dict):
    def __getitem__(self, name):
        accessed_resources_set.add(name)
        if not self.__contains__(name):
            raise NameError(f"name '{name}' is not defined")
        return self.get(name)

if __name__ == '__main__':
    PythonAgentExecutor().main()
<commit_msg>fixed 'get' function not getting the proper context<commit_after>import json
import sys
from jsonpath_ng import jsonpath, parse
import re
from io import StringIO
from lxml import etree

# noinspection PyMethodMayBeStatic
class PythonAgentExecutor(object):

    def __get_accessed_method(self, key):
        accessed_resources_set.add(key)

    def __disable_standard_io(self):
        old_io = (sys.stdin, sys.stdout, sys.stderr, sys.exit)
        sys.stdin, sys.stdout, sys.stderr, sys.exit = (None, None, None, lambda *x, **y: None)
        return old_io

    def __enable_standard_io(self, old_io):
        (sys.stdin, sys.stdout, sys.stderr, sys.exit) = old_io

    def cs_regex(self, str, regex):
        x = re.search(regex, str)
        return None if x is None else x.group()

    def cs_xpath_query(self, str, xpath):
        f = StringIO(str)
        tree = etree.parse(f)
        r = tree.xpath(xpath)
        return r[0] if len(r) > 0 else None

    def cs_json_query(self, str, json_path):
        json_data = json.loads(str)
        jsonpath_expr = parse(json_path)
        x = jsonpath_expr.find(json_data)
        return x[0].value if len(x) > 0 else None

    def __init_context(self, payload):
        global sys_prop
        global get_sp
        global get
        global cs_append
        global cs_prepend
        global cs_replace
        global cs_round
        global cs_extract_number
        global cs_substring
        global cs_to_upper
        global cs_to_lower
        global accessed
        global accessed_resources_set
        accessed_resources_set = set()
        context = payload["context"]

        if "sys_prop" in context:
            sys_prop = context["sys_prop"]
            accessed = self.__get_accessed_method

        env_setup = payload["envSetup"]
        get_sp = None
        get = None
        cs_append = None
        cs_prepend = None
        cs_replace = None
        cs_round = None
        cs_extract_number = None
        cs_substring = None
        cs_to_upper = None
        cs_to_lower = None
        exec (env_setup, globals())

    def main(self):
        global smaller_context
        try:
            raw_inputs = input().encode(sys.stdin.encoding).decode()
            payload = json.loads(raw_inputs)
            expression = payload["expression"]
            context = payload["context"]
            self.__init_context(payload)

            smaller_context = AccessAwareDict({"get_sp": get_sp, "get": get,
                                               "cs_append": cs_append,
                                               "cs_prepend": cs_prepend,
                                               "cs_replace": cs_replace,
                                               "cs_round": cs_round,
                                               "cs_extract_number": cs_extract_number,
                                               "cs_substring": cs_substring,
                                               "cs_to_upper": cs_to_upper,
                                               "cs_to_lower": cs_to_lower,
                                               "cs_regex": self.cs_regex,
                                               "cs_xpath_query": self.cs_xpath_query,
                                               "cs_json_query": self.cs_json_query,
                                               })

            for x in dir(__builtins__):
                smaller_context[x] = eval(x)

            for key, var in context.items():
                smaller_context[key] = var

            old_io = self.__disable_standard_io()

            try:
                expr_result = eval(expression, smaller_context)
                return_type = type(expr_result).__name__
                if return_type not in ['str', 'int', 'bool']:
                    return_type = 'str'
                final_result = {"returnResult": expr_result,
                                "accessedResources": list(accessed_resources_set),
                                "returnType": return_type}
            finally:
                self.__enable_standard_io(old_io)
        except Exception as e:
            final_result = {"exception": str(e)}

        print(json.dumps(final_result))

class AccessAwareDict(dict):
    def __getitem__(self, name):
        accessed_resources_set.add(name)
        if not self.__contains__(name):
            raise NameError(f"name '{name}' is not defined")
        return self.get(name)

if __name__ == '__main__':
    PythonAgentExecutor().main()
<|endoftext|>


<commit_before>import pyethereum
import shutil
import tempfile
import time
import logging
import sys
import spv

serpent = None

u = pyethereum.utils
t = pyethereum.transactions
b = pyethereum.blocks
pb = pyethereum.processblock

accounts = []
keys = []

for i in range(10):
    keys.append(u.sha3(str(i)))
    accounts.append(u.privtoaddr(keys[-1]))

k0, k1, k2, k3, k4, k5, k6, k7, k8, k9 = keys[:10]
a0, a1, a2, a3, a4, a5, a6, a7, a8, a9 = accounts[:10]

seed = 3**160

# Pseudo-RNG (deterministic for now for testing purposes)
def rand():
    global seed
    seed = pow(seed, 2, 2**512)
    return seed % 2**256

class state():
    def __init__(self, num_accounts=len(keys)):
        global serpent
        if not serpent:
            serpent = __import__('serpent')

        self.temp_data_dir = tempfile.mkdtemp()
        u.data_dir.set(self.temp_data_dir)

        o = {}
        for i in range(num_accounts):
            o[accounts[i]] = 10**24
        self.block = b.genesis(o)
        self.block.timestamp = 1410973349
        self.block.coinbase = a0

    def __del__(self):
        shutil.rmtree(self.temp_data_dir)

    def contract(self, code, sender=k0, endowment=0):
        evm = serpent.compile(code)
        return self.evm(evm, sender, endowment)

    def evm(self, evm, sender=k0, endowment=0):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        tx = t.contract(sendnonce, 1, gas_limit, endowment, evm)
        tx.sign(sender)
        (s, a) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Contract creation failed")
        return a

    def send(self, sender, to, value, data=[], funid=None, abi=None):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        (s, r) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Transaction failed")
        o = serpent.decode_datalist(r)
        return map(lambda x: x-2**256 if x > 2**255 else x, o)

    def profile(self, sender, to, value, data=[], funid=None, abi=None):
        tm, g = time.time(), self.block.gas_used
        o = self.send(sender, to, value, data, funid, abi)
        intrinsic_gas_used = pb.GTXDATA * len(self.last_tx.data) + pb.GTXCOST
        return {
            "time": time.time() - tm,
            "gas": self.block.gas_used - g - intrinsic_gas_used,
            "output": o
        }

    def mkspv(self, sender, to, value, data=[], funid=None, abi=None):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return spv.mk_transaction_spv_proof(self.block, tx)

    def verifyspv(self, sender, to, value, data=[], funid=None, abi=None, proof=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return spv.verify_transaction_spv_proof(self.block, tx, proof)

    def trace(self, sender, to, value, data=[]):

        # collect debug output
        log = []

        def log_receiver(data):
            log.append(data)

        pb.pblogger.log_op = 1
        pb.pblogger.log_stack = 1
        pb.pblogger.log_memory = 1
        pb.pblogger.log_storage = 1
        pb.pblogger.listeners.append(log_receiver)
        self.send(sender, to, value, data)
        pb.pblogger.listeners.remove(log_receiver)
        return log

    def mine(self, n=1, coinbase=a0):
        for i in range(n):
            self.block.finalize()
            t = self.block.timestamp + 6 + rand() % 12
            self.block = b.Block.init_from_parent(self.block, coinbase, '', t)

    def snapshot(self):
        return self.block.serialize()

    def revert(self, data):
        self.block = b.Block.deserialize(data)

def enable_logging():
    logging.basicConfig(level=logging.DEBUG, format='%(message)s')
    pb.pblogger.log_apply_op = True
    pb.pblogger.log_stack = True
    pb.pblogger.log_op = True

gas_limit = 100000
<commit_msg>Fixed signed representation of 2^255<commit_after>import pyethereum
import shutil
import tempfile
import time
import logging
import sys
import spv

serpent = None

u = pyethereum.utils
t = pyethereum.transactions
b = pyethereum.blocks
pb = pyethereum.processblock

accounts = []
keys = []

for i in range(10):
    keys.append(u.sha3(str(i)))
    accounts.append(u.privtoaddr(keys[-1]))

k0, k1, k2, k3, k4, k5, k6, k7, k8, k9 = keys[:10]
a0, a1, a2, a3, a4, a5, a6, a7, a8, a9 = accounts[:10]

seed = 3**160

# Pseudo-RNG (deterministic for now for testing purposes)
def rand():
    global seed
    seed = pow(seed, 2, 2**512)
    return seed % 2**256

class state():
    def __init__(self, num_accounts=len(keys)):
        global serpent
        if not serpent:
            serpent = __import__('serpent')

        self.temp_data_dir = tempfile.mkdtemp()
        u.data_dir.set(self.temp_data_dir)

        o = {}
        for i in range(num_accounts):
            o[accounts[i]] = 10**24
        self.block = b.genesis(o)
        self.block.timestamp = 1410973349
        self.block.coinbase = a0

    def __del__(self):
        shutil.rmtree(self.temp_data_dir)

    def contract(self, code, sender=k0, endowment=0):
        evm = serpent.compile(code)
        return self.evm(evm, sender, endowment)

    def evm(self, evm, sender=k0, endowment=0):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        tx = t.contract(sendnonce, 1, gas_limit, endowment, evm)
        tx.sign(sender)
        (s, a) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Contract creation failed")
        return a

    def send(self, sender, to, value, data=[], funid=None, abi=None):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        (s, r) = pb.apply_transaction(self.block, tx)
        if not s:
            raise Exception("Transaction failed")
        o = serpent.decode_datalist(r)
        return map(lambda x: x-2**256 if x >= 2**255 else x, o)

    def profile(self, sender, to, value, data=[], funid=None, abi=None):
        tm, g = time.time(), self.block.gas_used
        o = self.send(sender, to, value, data, funid, abi)
        intrinsic_gas_used = pb.GTXDATA * len(self.last_tx.data) + pb.GTXCOST
        return {
            "time": time.time() - tm,
            "gas": self.block.gas_used - g - intrinsic_gas_used,
            "output": o
        }

    def mkspv(self, sender, to, value, data=[], funid=None, abi=None):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return spv.mk_transaction_spv_proof(self.block, tx)

    def verifyspv(self, sender, to, value, data=[], funid=None, abi=None, proof=[]):
        sendnonce = self.block.get_nonce(u.privtoaddr(sender))
        if funid is not None:
            evmdata = serpent.encode_abi(funid, abi)
        else:
            evmdata = serpent.encode_datalist(*data)
        tx = t.Transaction(sendnonce, 1, gas_limit, to, value, evmdata)
        self.last_tx = tx
        tx.sign(sender)
        return spv.verify_transaction_spv_proof(self.block, tx, proof)

    def trace(self, sender, to, value, data=[]):

        # collect debug output
        log = []

        def log_receiver(data):
            log.append(data)

        pb.pblogger.log_op = 1
        pb.pblogger.log_stack = 1
        pb.pblogger.log_memory = 1
        pb.pblogger.log_storage = 1
        pb.pblogger.listeners.append(log_receiver)
        self.send(sender, to, value, data)
        pb.pblogger.listeners.remove(log_receiver)
        return log

    def mine(self, n=1, coinbase=a0):
        for i in range(n):
            self.block.finalize()
            t = self.block.timestamp + 6 + rand() % 12
            self.block = b.Block.init_from_parent(self.block, coinbase, '', t)

    def snapshot(self):
        return self.block.serialize()

    def revert(self, data):
        self.block = b.Block.deserialize(data)

def enable_logging():
    logging.basicConfig(level=logging.DEBUG, format='%(message)s')
    pb.pblogger.log_apply_op = True
    pb.pblogger.log_stack = True
    pb.pblogger.log_op = True

gas_limit = 100000
<|endoftext|>


<commit_before>
'''
Routines for feathering large cubes together using uvcombine.

Some of these *should* be moved into uvcombine in good time.
'''

from spectral_cube import SpectralCube
from uvcombine.uvcombine import feather_simple, feather_compare
from astropy import log
import astropy.units as u

import numpy as np

from .io_utils import create_huge_fits
from .progressbar import _map_context, ProgressBar

def feather_cube(cube_hi, cube_lo, verbose=True, save_feather=True,
                 save_name=None, num_cores=1, chunk=100,
                 frequency=1.42040575177 * u.GHz,
                 feather_kwargs={}):
    '''
    Feather two cubes together. The spectral axis of the cubes *must match*.
    This function is specifically for cube too large to fit in memory. The
    feathered planes are saved directly to a new FITS file.
    '''

    # Ensure the spectral axes are the same.
    if not np.allclose(cube_hi.spectral_axis.to(cube_lo._spectral_unit).value,
                       cube_lo.spectral_axis.value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    if save_feather:
        if save_name is None:
            raise TypeError("save_name must be given the save the shifted "
                            "cube.")

        output_hdu = create_huge_fits(save_name, cube_hi.header,
                                      return_hdu=True)
    else:
        newcube = np.empty(cube_hi.shape, dtype=cube_hi[:1, 0, 0].dtype)

    num_chans = cube_hi.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube_hi[chan:chan + 1], cube_lo[chan:chan + 1],
                    frequency, feather_kwargs) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose)as map:
            log.info("Feathering")
            output = map(_feather, changen)

        for chan, shifted_arr in output:
            if save_feather:
                output_hdu[0].data[chan] = shifted_arr.real
            else:
                newcube[chan] = shifted_arr.real

        if save_feather:
            output_hdu.flush()

    if save_feather:

        # Append the beam table, if needed
        if hasattr(cube_hi, 'beams'):
            from spectral_cube.cube_utils import beams_to_bintable
            output_hdu.append(beams_to_bintable(cube_hi.beams))

        output_hdu.close()
    else:
        return SpectralCube(newcube, wcs=cube_hi.wcs,
                            header=cube_hi.header, meta=cube_hi.meta)

def _feather(args):
    '''
    Feather 2D images together.
    '''
    chan, plane_hi, plane_lo, freq, kwargs = args

    # TODO: When Jy/beam is supported in SpectralCube, just let uvcombine
    # match the units. BUT we also need Slice objects to retain a 1 element
    # spectral axis!
    if plane_hi.unit == u.K:
        if hasattr(plane_hi, 'beams'):
            beam = plane_hi.beams[0]
        else:
            beam = plane_hi.beam

        plane_hi = plane_hi.to(u.Jy, plane_hi.beam.jtok_equiv(freq))
    plane_hi._unit = u.Jy / u.beam

    if plane_lo.unit == u.K:
        if hasattr(plane_lo, 'beams'):
            beam = plane_lo.beams[0]
        else:
            beam = plane_lo.beam

        plane_lo = plane_lo.to(u.Jy, beam.jtok_equiv(freq))
    plane_lo._unit = u.Jy / u.beam

    if hasattr(plane_hi, 'hdu'):
        plane_hi_hdu = plane_hi.hdu
    else:
        plane_hi_hdu = plane_hi.hdulist[0]
        plane_hi_hdu.header.update(plane_hi.beams[0].to_header_keywords())

    if hasattr(plane_lo, 'hdu'):
        plane_lo_hdu = plane_lo.hdu
    else:
        plane_lo_hdu = plane_lo.hdulist[0]
        plane_lo_hdu.header.update(plane_lo.beams[0].to_header_keywords())

    feathered = feather_simple(plane_hi_hdu, plane_lo_hdu, **kwargs)

    # Expect that the interferometer image will cover a smaller region.
    # Cut to that region.
    feathered[np.isnan(plane_hi[0])] = np.NaN

    return chan, feathered

def feather_compare_cube(cube_hi, cube_lo, LAS, lowresfwhm=None,
                         verbose=True, num_cores=1, chunk=100):
    '''
    Record the ratios of the flux in the overlap region between the cubes.
    '''

    # Ensure the spectral axes are the same.
    if not np.allclose(cube_hi.spectral_axis.to(cube_lo._spectral_unit).value,
                       cube_lo.spectral_axis.value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    num_chans = cube_hi.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]
    if chunked_channels[0].size == 0:
        chunked_channels = chunked_channels[1:]

    radii = []
    ratios = []
    highres_pts = []
    lowres_pts = []

    if lowresfwhm is None:
        lowresfwhm = cube_lo.beam.major.to(u.arcsec)
    else:
        lowresfwhm = lowresfwhm.to(u.arcsec)

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube_hi[chan], cube_lo[chan],
                    LAS, lowresfwhm) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose)as map:
            output = map(_compare, changen)

        chan_out = np.array([out[0] for out in output])

        for jj in chan_out.argsort():

            radii.append(output[jj][1][0])
            ratios.append(output[jj][1][1])
            highres_pts.append(output[jj][1][2])
            lowres_pts.append(output[jj][1][3])

    return radii, ratios, highres_pts, lowres_pts

def _compare(args):

    chan, plane_hi, plane_lo, LAS, lowresfwhm = args

    hi_beam = plane_hi.beam

    out = feather_compare(plane_hi.to(u.K, hi_beam.jtok_equiv(1.42040575177 * u.GHz)).hdu,
                          plane_lo.hdu,
                          return_samples=True, doplot=False,
                          LAS=LAS, SAS=lowresfwhm,
                          lowresfwhm=lowresfwhm)

    return chan, out
<commit_msg>Minor updates<commit_after>
'''
Routines for feathering large cubes together using uvcombine.

Some of these *should* be moved into uvcombine in good time.
'''

from spectral_cube import SpectralCube
from uvcombine.uvcombine import feather_simple, feather_compare
from astropy import log
import astropy.units as u

import numpy as np

from .io_utils import create_huge_fits
from .progressbar import _map_context

def feather_cube(cube_hi, cube_lo, verbose=True, save_feather=True,
                 save_name=None, num_cores=1, chunk=100,
                 frequency=1.42040575177 * u.GHz,
                 feather_kwargs={}):
    '''
    Feather two cubes together. The spectral axis of the cubes *must match*.
    This function is specifically for cube too large to fit in memory. The
    feathered planes are saved directly to a new FITS file.
    '''

    # Ensure the spectral axes are the same.
    if not np.allclose(cube_hi.spectral_axis.to(cube_lo._spectral_unit).value,
                       cube_lo.spectral_axis.value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    if save_feather:
        if save_name is None:
            raise TypeError("save_name must be given the save the shifted "
                            "cube.")

        output_hdu = create_huge_fits(save_name, cube_hi.header,
                                      return_hdu=True)
    else:
        newcube = np.empty(cube_hi.shape, dtype=cube_hi[:1, 0, 0].dtype)

    num_chans = cube_hi.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube_hi[chan:chan + 1], cube_lo[chan:chan + 1],
                    frequency, feather_kwargs) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose) as map:
            log.info("Feathering")
            output = map(_feather, changen)

        for chan, shifted_arr in output:
            if save_feather:
                output_hdu[0].data[chan] = shifted_arr.real
            else:
                newcube[chan] = shifted_arr.real

        if save_feather:
            output_hdu.flush()

    if save_feather:

        # Append the beam table, if needed
        if hasattr(cube_hi, 'beams'):
            from spectral_cube.cube_utils import beams_to_bintable
            output_hdu.append(beams_to_bintable(cube_hi.beams))

        output_hdu.close()
    else:
        return SpectralCube(newcube, wcs=cube_hi.wcs,
                            header=cube_hi.header, meta=cube_hi.meta)

def _feather(args):
    '''
    Feather 2D images together.
    '''
    chan, plane_hi, plane_lo, freq, kwargs = args

    # TODO: When Jy/beam is supported in SpectralCube, just let uvcombine
    # match the units. BUT we also need Slice objects to retain a 1 element
    # spectral axis!
    if plane_hi.unit == u.K:
        if hasattr(plane_hi, 'beams'):
            beam = plane_hi.beams[0]
        else:
            beam = plane_hi.beam

        plane_hi = plane_hi.to(u.Jy, plane_hi.beam.jtok_equiv(freq))
    plane_hi._unit = u.Jy / u.beam

    if plane_lo.unit == u.K:
        if hasattr(plane_lo, 'beams'):
            beam = plane_lo.beams[0]
        else:
            beam = plane_lo.beam

        plane_lo = plane_lo.to(u.Jy, beam.jtok_equiv(freq))
    plane_lo._unit = u.Jy / u.beam

    if hasattr(plane_hi, 'hdu'):
        plane_hi_hdu = plane_hi.hdu
    else:
        plane_hi_hdu = plane_hi.hdulist[0]
        plane_hi_hdu.header.update(plane_hi.beams[0].to_header_keywords())

    if hasattr(plane_lo, 'hdu'):
        plane_lo_hdu = plane_lo.hdu
    else:
        plane_lo_hdu = plane_lo.hdulist[0]
        plane_lo_hdu.header.update(plane_lo.beams[0].to_header_keywords())

    feathered = feather_simple(plane_hi_hdu, plane_lo_hdu, **kwargs)

    # Expect that the interferometer image will cover a smaller region.
    # Cut to that region.
    feathered[np.isnan(plane_hi[0])] = np.NaN

    return chan, feathered

def feather_compare_cube(cube_hi, cube_lo, LAS, lowresfwhm=None,
                         frequency=1.42040575177 * u.GHz,
                         verbose=True, num_cores=1, chunk=100):
    '''
    Record the ratios of the flux in the overlap region between the cubes.
    '''

    # Ensure the spectral axes are the same.
    if not np.allclose(cube_hi.spectral_axis.to(cube_lo._spectral_unit).value,
                       cube_lo.spectral_axis.value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    num_chans = cube_hi.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]
    if chunked_channels[0].size == 0:
        chunked_channels = chunked_channels[1:]

    radii = []
    ratios = []
    highres_pts = []
    lowres_pts = []

    if lowresfwhm is None:
        lowresfwhm = cube_lo.beam.major.to(u.arcsec)
    else:
        lowresfwhm = lowresfwhm.to(u.arcsec)

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube_hi[chan], cube_lo[chan],
                    LAS, lowresfwhm, frequency) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose)as map:
            output = map(_compare, changen)

        chan_out = np.array([out[0] for out in output])

        for jj in chan_out.argsort():

            radii.append(output[jj][1][0])
            ratios.append(output[jj][1][1])
            highres_pts.append(output[jj][1][2])
            lowres_pts.append(output[jj][1][3])

    return radii, ratios, highres_pts, lowres_pts

def _compare(args):

    chan, plane_hi, plane_lo, LAS, lowresfwhm, freq = args

    hi_beam = plane_hi.beam

    out = feather_compare(plane_hi.to(u.K, hi_beam.jtok_equiv(freq)).hdu,
                          plane_lo.hdu,
                          return_samples=True, doplot=False,
                          LAS=LAS, SAS=lowresfwhm,
                          lowresfwhm=lowresfwhm)

    return chan, out
<|endoftext|>