<commit_before>from __future__ import absolute_import

"""This module offers a display and interaction frontend with Qt.

It will try importing PySide first, and if that fails PyQt. The code will
constantly be tested with both bindings."""

from .ca import binRule

try:
    from PySide.QtCore import *
    from PySide.QtGui import *
    print "using pyside"
except ImportError:
    from PyQt4.QtCore import *
    from PyQt4.QtGui import *
    print "using pyqt4"
import Queue
import sys
import random
import time

class Control(QWidget):
    """Control a simulator with buttons or from the interactive console."""
    def __init__(self, simulator, parent=None):
        """:param simulator: The simulator object to control.
        :param parent: the QWidget to set as parent."""
        super(Control, self).__init__(parent)

        self.sim = simulator
        self.timer_delay = 10
        self.attached_displays = []

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("Start")
        self.stop_button = QPushButton("Stop")
        self.stop_button.setDisabled(True)
        delay = QSpinBox()
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)
        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.setDisabled(True)
        self.stop_button.setEnabled(True)

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.setDisabled(True)
        self.start_button.setEnabled(True)

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()
        for d in self.attached_displays:
            d.after_step()

    def attach_display(self, display):
        """Attach a display to the control.

        Those displays are updated whenever a step occurs."""
        self.attached_displays.append(display)

    def detach_display(self, display):
        """Detach an attached display from the control."""
        self.attached_displays.remove(display)

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

class BaseDisplay(QWidget):
    """A base class for different types of displays.

    Manages the config display queue, scrolling, ..."""
    def __init__(self, width, height, queue_size=1, scale=1, **kwargs):
        """Initialize the BaseDisplay.

        :param width: The width of the image to build.
        :param height: The height of the image to build.
        :param queue_size: The amount of histories that may pile up before
                           forcing a redraw.
        """
        super(BaseDisplay, self).__init__(**kwargs)
        self.img_width, self.img_height = width, height
        self.img_scale = scale

        self.resize(self.img_width * self.img_scale,
                    self.img_height * self.img_scale)

        self.create_image_surf()
        self.display_queue = Queue.Queue(queue_size)

        self.last_step = 0
        self.queued_steps = 0

    def create_image_surf(self):
        """Create the image surface when the display is created."""
        self.image = QBitmap(self.img_width, self.img_height)
        self.image.clear()

class HistoryDisplay(BaseDisplay):
    """A Display that displays one-dimensional cellular automatons by drawing
    consecutive configurations below each other, wrapping around to the top."""
    def __init__(self, simulator, size, scale=1, **kwargs):
        """:param simulator: The simulator to use.
        :param size: The amount of lines of history to keep before wrapping.
        :param scale: The size of each pixel.
        :param parent: The QWidget to set as parent."""
        super(HistoryDisplay, self).__init__(width=simulator.sizeX,
                      height=size,
                      queue_size=size,
                      scale=scale,
                      **kwargs)
        self.sim = simulator

    def paintEvent(self, ev):
        """Get new configurations, update the internal pixmap, refresh the
        display.

        This is called from Qt whenever a repaint is in order. Do not call it
        yourself. :meth:`after_step` will call update, which will trigger a
        :meth:`paintEvent`.

        .. note::
            In order not to turn into an endless loop, update at most 100
            lines or :attr:`size` lines, whichever is lower."""
        rendered = 0
        y = self.last_step % self.img_height
        paint = QPainter(self.image)
        try:
            while rendered < min(100, self.img_height):
                conf = self.display_queue.get_nowait()
                self.queued_steps -= 1
                ones = []
                paint.setPen(Qt.color0)
                for x, field in enumerate(conf):
                    if field == 1:
                        ones.append(x)
                    else:
                        paint.drawPoint(x, y)
                paint.setPen(Qt.color1)
                for x in ones:
                    paint.drawPoint(x, y)
                rendered += 1
                self.last_step += 1
                y = self.last_step % self.img_height
        except Queue.Empty:
            pass

        copier = QPainter(self)
        copier.setPen(QColor("white"))
        copier.setBackground(QColor("black"))
        copier.setBackgroundMode(Qt.OpaqueMode)

        self._copy_pixmap(ev.rect(), copier)

    def _copy_pixmap(self, target, painter, damageRect=None):
        """Cleverly copy over a part of the stored image onto the widget.

        :param target: The rect on the widget that's supposed to be updated.
        :param painter: The painter to be used for drawing.
        :param damageRect: An optional rect to limit the redrawing."""
        if damageRect:
            target = damageRect.intersected(target)
        if self.img_scale == 1:
            src = target
        else:
            target.setX(int(target.x() / self.img_scale) * self.img_scale)
            target.setY(int(target.y() / self.img_scale) * self.img_scale)
            target.setWidth(int(target.width() / self.img_scale) * self.img_scale + self.img_scale)
            target.setHeight(int(target.height() / self.img_scale) * self.img_scale + self.img_scale)
            src = QRect(target.x() / self.img_scale,
                        target.y() / self.img_scale,
                        target.width() / self.img_scale,
                        target.height() / self.img_scale)

        painter.drawPixmap(target, self.image, src)

    def after_step(self):
        """React to a single step. Copy the current configuration into the
        :attr:`display_queue` and schedule a call to :meth:`paintEvent`."""
        conf = self.sim.getConf().copy()
        try:
            self.display_queue.put_nowait(conf)
        except Queue.Full:
            # the queue is initialised to hold as much lines as the
            # screen does. if we have more changes queued up than can
            # fit on the screen, we can just skip some of them.
            self.display_queue.get()
            self.display_queue.put(conf)
            # theoretically, in this case, the queued steps var would
            # have to be treated in a special way so that the update
            # line wanders around so that it still works, but
            # since the whole screen will get updated anyway, we
            # don't need to care at all.

        self.queued_steps += 1
        self.update(QRect(
            QPoint(0, ((self.last_step + self.queued_steps - 1) % self.img_height) * self.img_scale),
            QSize(self.img_width * self.img_scale, self.img_scale)))

def main():
    app = QApplication(sys.argv)

    scale = 2
    sizex, sizey = 800 / scale, 600 / scale

    # get a random beautiful CA
    sim = binRule(random.choice(
         [22, 26, 30, 45, 60, 73, 90, 105, 110, 122, 106, 150]),
         sizex, 0, binRule.INIT_RAND)
    disp = HistoryDisplay(sim, sizey, scale)

    window = QMainWindow()

    central_widget = QWidget(window)

    window_l = QVBoxLayout(central_widget)

    scroller = QScrollArea()
    window_l.addWidget(scroller)
    scroller.setWidget(disp)
    scroller.resize(800, 600)

    control = Control(sim, parent=central_widget)
    window_l.addWidget(control)
    control.attach_display(disp)

    window.setCentralWidget(central_widget)
    window.show()

    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<commit_msg>make HistoryDisplay a bit faster<commit_after>from __future__ import absolute_import

"""This module offers a display and interaction frontend with Qt.

It will try importing PySide first, and if that fails PyQt. The code will
constantly be tested with both bindings."""

from .ca import binRule

try:
    from PySide.QtCore import *
    from PySide.QtGui import *
    print "using pyside"
except ImportError:
    from PyQt4.QtCore import *
    from PyQt4.QtGui import *
    print "using pyqt4"
import Queue
import sys
import random
import time

class Control(QWidget):
    """Control a simulator with buttons or from the interactive console."""
    def __init__(self, simulator, parent=None):
        """:param simulator: The simulator object to control.
        :param parent: the QWidget to set as parent."""
        super(Control, self).__init__(parent)

        self.sim = simulator
        self.timer_delay = 10
        self.attached_displays = []

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("Start")
        self.stop_button = QPushButton("Stop")
        self.stop_button.setDisabled(True)
        delay = QSpinBox()
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)
        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.setDisabled(True)
        self.stop_button.setEnabled(True)

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.setDisabled(True)
        self.start_button.setEnabled(True)

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()
        for d in self.attached_displays:
            d.after_step()

    def attach_display(self, display):
        """Attach a display to the control.

        Those displays are updated whenever a step occurs."""
        self.attached_displays.append(display)

    def detach_display(self, display):
        """Detach an attached display from the control."""
        self.attached_displays.remove(display)

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

class BaseDisplay(QWidget):
    """A base class for different types of displays.

    Manages the config display queue, scrolling, ..."""
    def __init__(self, width, height, queue_size=1, scale=1, **kwargs):
        """Initialize the BaseDisplay.

        :param width: The width of the image to build.
        :param height: The height of the image to build.
        :param queue_size: The amount of histories that may pile up before
                           forcing a redraw.
        """
        super(BaseDisplay, self).__init__(**kwargs)
        self.img_width, self.img_height = width, height
        self.img_scale = scale

        self.resize(self.img_width * self.img_scale,
                    self.img_height * self.img_scale)

        self.create_image_surf()
        self.display_queue = Queue.Queue(queue_size)

        self.last_step = 0
        self.queued_steps = 0

    def create_image_surf(self):
        """Create the image surface when the display is created."""
        self.image = QImage(self.img_width * self.img_scale,
                            self.img_height * self.img_scale,
                            QImage.Format_RGB444)
        self.image.fill(0)

class HistoryDisplay(BaseDisplay):
    """A Display that displays one-dimensional cellular automatons by drawing
    consecutive configurations below each other, wrapping around to the top."""
    def __init__(self, simulator, size, scale=1, **kwargs):
        """:param simulator: The simulator to use.
        :param size: The amount of lines of history to keep before wrapping.
        :param scale: The size of each pixel.
        :param parent: The QWidget to set as parent."""
        super(HistoryDisplay, self).__init__(width=simulator.sizeX,
                      height=size,
                      queue_size=size,
                      scale=scale,
                      **kwargs)
        self.sim = simulator

    def paintEvent(self, ev):
        """Get new configurations, update the internal pixmap, refresh the
        display.

        This is called from Qt whenever a repaint is in order. Do not call it
        yourself. :meth:`after_step` will call update, which will trigger a
        :meth:`paintEvent`.

        .. note::
            In order not to turn into an endless loop, update at most 100
            lines or :attr:`size` lines, whichever is lower."""
        rendered = 0
        y = self.last_step % self.img_height
        w = self.img_width
        scale = self.img_scale
        try:
            painter = QPainter(self.image)
            while rendered < min(100, self.img_height):
                conf = self.display_queue.get_nowait()
                nconf = np.empty((w, 1, 2), np.uint8, "C")
                nconf[...,0,0] = conf * 255
                nconf[...,1] = nconf[...,0]

                image = QImage(nconf.data, w, 1, QImage.Format_RGB444).scaled(w * scale, scale)
                painter.drawImage(QPoint(0, y * scale), image)

                self.queued_steps -= 1
                rendered += 1
                self.last_step += 1
                y = self.last_step % self.img_height
        except Queue.Empty:
            pass
        finally:
            del painter

        copier = QPainter(self)
        copier.drawImage(QPoint(0, 0), self.image)

        del copier

    def after_step(self):
        """React to a single step. Copy the current configuration into the
        :attr:`display_queue` and schedule a call to :meth:`paintEvent`."""
        conf = self.sim.getConf().copy()
        try:
            self.display_queue.put_nowait(conf)
        except Queue.Full:
            # the queue is initialised to hold as much lines as the
            # screen does. if we have more changes queued up than can
            # fit on the screen, we can just skip some of them.
            self.display_queue.get()
            self.display_queue.put(conf)
            # theoretically, in this case, the queued steps var would
            # have to be treated in a special way so that the update
            # line wanders around so that it still works, but
            # since the whole screen will get updated anyway, we
            # don't need to care at all.

        self.queued_steps += 1
        self.update(QRect(
            QPoint(0, ((self.last_step + self.queued_steps - 1) % self.img_height) * self.img_scale),
            QSize(self.img_width * self.img_scale, self.img_scale)))

def main():
    app = QApplication(sys.argv)

    scale = 2
    w, h = 300, 300

    # get a random beautiful CA
    sim = binRule(random.choice(
         [22, 26, 30, 45, 60, 73, 90, 105, 110, 122, 106, 150]),
         w, 0, binRule.INIT_RAND)

    disp = HistoryDisplay(sim, h, scale)
    disp.after_step()

    window = QMainWindow()

    central_widget = QWidget(window)

    window_l = QVBoxLayout(central_widget)

    scroller = QScrollArea()
    window_l.addWidget(scroller)
    scroller.setWidget(disp)
    scroller.resize(800, 600)

    control = Control(sim, parent=central_widget)
    window_l.addWidget(control)
    control.attach_display(disp)

    window.setCentralWidget(central_widget)
    window.show()

    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import httplib2
from urllib import urlencode

# Note: I've seen this pattern for dealing with json in different versions of
# python in a lot of modules -- if there's a better way, I'd love to use it.
try:
    # python 2.6 and greater
    import json
except ImportError:
    try:
        # python 2.5
        import simplejson
        json = simplejson
    except ImportError:
        # if we're in django or Google AppEngine land
        # use this as a last resort
        from django.utils import simplejson
        json = simplejson

class ZencoderError(Exception):
    pass

class ZencoderResponseError(Exception):
    def __init__(self, http_response, content):
        self.http_response = http_response
        self.content = content

class HTTPBackend(object):
    """
    Abstracts out an HTTP backend, but defaults to httplib2. Required arguments
    are `base_url` and `api_key`.

    .. note::
       While `as_xml` is provided as a keyword argument, XML or input or output
       is not supported.
    """
    def __init__(self, base_url, api_key, as_xml=False, resource_name=None, timeout=None, test=False, version=None):
        self.base_url = base_url
        if resource_name:
            self.base_url = self.base_url + resource_name

        self.http = httplib2.Http(timeout=timeout)
        self.as_xml = as_xml
        self.api_key = api_key
        self.test = test
        self.version = version

    def content_length(self, body):
        """
        Returns the content length as an int for the given `body` data. Used by
        PUT and POST requests to set the Content-Length header.
        """
        return str(len(body)) if body else "0"

    @property
    def headers(self):
        """ Returns default headers, by setting the Content-Type and Accepts
        headers.
        """
        if self.as_xml:
            return {'Content-Type': 'application/xml',
                    'Accepts': 'application/xml'}
        else:
            return {'Content-Type': 'application/json',
                    'Accepts': 'application/json'}

    def encode(self, data):
        """
        Encodes data as JSON (by calling `json.dumps`), so that it can be
        passed onto the Zencoder API.

        .. note::
           Encoding as XML is not supported.
        """
        if not self.as_xml:
            return json.dumps(data)
        else:
            raise NotImplementedError('Encoding as XML is not supported.')

    def decode(self, raw_body):
        """
        Returns the JSON-encoded `raw_body` and decodes it to a `dict` (using
        `json.loads`).

        .. note::
           Decoding as XML is not supported.
        """
        if not self.as_xml:
            # only parse json when it exists, else just return None
            if not raw_body or raw_body == ' ':
                return None
            else:
                return json.loads(raw_body)
        else:
            raise NotImplementedError('Decoding as XML is not supported.')

    def delete(self, url, params=None):
        """
        Executes an HTTP DELETE request for the given URL

        params should be a urllib.urlencoded string
        """
        if params:
            url = '?'.join([url, params])

        response, content = self.http.request(url, method="DELETE",
                                              headers=self.headers)
        return self.process(response, content)

    def get(self, url, data=None):
        """
        Executes an HTTP GET request for the given URL

        data should be a dictionary of url parameters
        """
        if data:
            params = urlencode(data)
            url = '?'.join([url, params])
        response, content = self.http.request(url, method="GET",
                                              headers=self.headers)
        return self.process(response, content)

    def post(self, url, body=None):
        """
        Executes an HTTP POST request for the given URL
        """
        headers = self.headers
        headers['Content-Length'] = self.content_length(body)
        response, content = self.http.request(url, method="POST",
                                              body=body,
                                              headers=self.headers)

        return self.process(response, content)

    def put(self, url, data=None, body=None):
        """
        Executes an HTTP PUT request for the given URL
        """
        headers = self.headers
        headers['Content-Length'] = self.content_length(body)

        if data:
            params = urlencode(data)
            url = '?'.join([url, params])

        response, content = self.http.request(url, method="PUT",
                                              body=body,
                                              headers=headers)

        return self.process(response, content)

    def process(self, http_response, content):
        """
        Returns HTTP backend agnostic Response data
        """

        try:
            code = http_response.status
            body = self.decode(content)
            response = Response(code, body, content, http_response)

            return response

        except ValueError:
            raise ZencoderResponseError(http_response, content)

class Zencoder(object):
    """ This is the entry point to the Zencoder API """
    def __init__(self, api_key=None, api_version=None, as_xml=False, timeout=None, test=False):
        """
        Initializes Zencoder. You must have a valid API_KEY.

        You can pass in the api_key as an argument, or set
        `ZENCODER_API_KEY` as an environment variable, and it will use
        that, if api_key is unspecified.

        Set api_version='edge' to get the Zencoder development API. (defaults to 'v2')
        Set as_xml=True to get back xml data instead of the default json.
        """
        if not api_version:
            api_version = 'v2'

        self.base_url = 'https://app.zencoder.com/api/'
        if not api_version == 'edge':
            self.base_url = self.base_url + '%s/' % api_version

        if not api_key:
            try:
                self.api_key = os.environ['ZENCODER_API_KEY']
            except KeyError:
                raise ZencoderError('ZENCODER_API_KEY not set')
        else:
            self.api_key = api_key

        self.test = test
        self.as_xml = as_xml

        args = (self.base_url, self.api_key, self.as_xml)
        kwargs = dict(timeout=timeout, test=self.test, version=api_version)
        self.job = Job(*args, **kwargs)
        self.account = Account(*args, **kwargs)
        self.output = Output(*args, **kwargs)
        self.report = None
        if api_version == 'v2':
            self.report = Report(*args, **kwargs)

class Response(object):
    """
    The Response object stores the details of an API request in an XML/JSON
    agnostic way.
    """
    def __init__(self, code, body, raw_body, raw_response):
        self.code = code
        self.body = body
        self.raw_body = raw_body
        self.raw_response = raw_response

class Account(HTTPBackend):
    """ Account object """
    def __init__(self, *args, **kwargs):
        """
        Initializes an Account object
        """
        kwargs['resource_name'] = 'account'
        super(Account, self).__init__(*args, **kwargs)

    def create(self, email, tos=1, options=None):
        """
        Creates an account with Zencoder, no API Key necessary.
        """
        data = {'email': email,
                'terms_of_service': str(tos)}
        if options:
            data.update(options)

        return self.post(self.base_url, body=self.encode(data))

    def details(self):
        """
        Gets your account details.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url, data=data)

    def integration(self):
        """
        Puts your account into integration mode.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url + '/integration', data=data)

    def live(self):
        """
        Puts your account into live mode.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url + '/live', data=data)

class Output(HTTPBackend):
    """ Gets information regarding outputs """
    def __init__(self, *args, **kwargs):
        """
        Contains all API methods relating to Outputs.
        """
        kwargs['resource_name'] = 'outputs'
        super(Output, self).__init__(*args, **kwargs)

    def progress(self, output_id):
        """
        Gets the given output id's progress.
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s/progress' % str(output_id),
                        data=data)

    def details(self, output_id):
        """
        Gets the given output id's details
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s' % str(output_id),
                        data=data)

class Job(HTTPBackend):
    """
    Contains all API methods relating to transcoding Jobs.
    """
    def __init__(self, *args, **kwargs):
        """
        Initializes a job object
        """
        kwargs['resource_name'] = 'jobs'
        super(Job, self).__init__(*args, **kwargs)

    def create(self, input, outputs=None, options=None):
        """
        Creates a job

        @param input: the input url as string
        @param outputs: a list of output dictionaries
        @param options: a dictionary of job options
        """
        as_test = int(self.test)

        data = {"api_key": self.api_key, "input": input, "test": as_test}
        if outputs:
            data['outputs'] = outputs

        if options:
            data.update(options)

        return self.post(self.base_url, body=self.encode(data))

    def list(self, page=1, per_page=50):
        """
        Lists some jobs.

        @param page: <int> the page of results to return
        @param per_page: <int> the number of results per page
        """
        data = {"api_key": self.api_key,
                "page": page,
                "per_page": per_page}
        return self.get(self.base_url, data=data)

    def details(self, job_id):
        """
        Gets details for the given job
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s' % str(job_id), data=data)

    def progress(self, job_id):
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s/progress' % str(job_id), data=data)

    def resubmit(self, job_id):
        """
        Resubmits the given `job_id`
        """
        data = {'api_key': self.api_key}
        url = self.base_url + '/%s/resubmit' % str(job_id)
        return self.put(url, data=data)

    def cancel(self, job_id):
        """
        Cancels the given `job_id`
        """
        if self.version == 'v1':
            verb = self.get
        else:
            verb = self.put

        data = {'api_key': self.api_key}
        url = self.base_url + '/%s/cancel' % str(job_id)
        return verb(url, data=data)

    def delete(self, job_id):
        """
        Deletes the given `job_id`

        WARNING: This method is aliased to `Job.cancel` -- it is deprecated in
                 API version 2 and greater.
        """
        return self.cancel(job_id)

class Report(HTTPBackend):
    def __init__(self, *args, **kwargs):
        """
        Contains all API methods relating to Outputs.
        """
        kwargs['resource_name'] = 'reports'
        super(Report, self).__init__(*args, **kwargs)

    def details(self, start_date=None, end_date=None, grouping=None):
        """
        Gets a detailed Report
        """
        data = {'api_key': self.api_key}
        if start_date:
            data['from'] = start_date

        if end_date:
            data['to'] = end_date

        if grouping:
            data['grouping'] = grouping

        url = self.base_url + '/minutes'
        return self.get(url, data=data)
<commit_msg>Re-added linebreak<commit_after>import os
import httplib2
from urllib import urlencode

# Note: I've seen this pattern for dealing with json in different versions of
# python in a lot of modules -- if there's a better way, I'd love to use it.
try:
    # python 2.6 and greater
    import json
except ImportError:
    try:
        # python 2.5
        import simplejson
        json = simplejson
    except ImportError:
        # if we're in django or Google AppEngine land
        # use this as a last resort
        from django.utils import simplejson
        json = simplejson

class ZencoderError(Exception):
    pass

class ZencoderResponseError(Exception):
    def __init__(self, http_response, content):
        self.http_response = http_response
        self.content = content

class HTTPBackend(object):
    """
    Abstracts out an HTTP backend, but defaults to httplib2. Required arguments
    are `base_url` and `api_key`.

    .. note::
       While `as_xml` is provided as a keyword argument, XML or input or output
       is not supported.
    """
    def __init__(self, base_url, api_key, as_xml=False, resource_name=None, timeout=None, test=False, version=None):
        self.base_url = base_url
        if resource_name:
            self.base_url = self.base_url + resource_name

        self.http = httplib2.Http(timeout=timeout)
        self.as_xml = as_xml
        self.api_key = api_key
        self.test = test
        self.version = version

    def content_length(self, body):
        """
        Returns the content length as an int for the given `body` data. Used by
        PUT and POST requests to set the Content-Length header.
        """
        return str(len(body)) if body else "0"

    @property
    def headers(self):
        """ Returns default headers, by setting the Content-Type and Accepts
        headers.
        """
        if self.as_xml:
            return {'Content-Type': 'application/xml',
                    'Accepts': 'application/xml'}
        else:
            return {'Content-Type': 'application/json',
                    'Accepts': 'application/json'}

    def encode(self, data):
        """
        Encodes data as JSON (by calling `json.dumps`), so that it can be
        passed onto the Zencoder API.

        .. note::
           Encoding as XML is not supported.
        """
        if not self.as_xml:
            return json.dumps(data)
        else:
            raise NotImplementedError('Encoding as XML is not supported.')

    def decode(self, raw_body):
        """
        Returns the JSON-encoded `raw_body` and decodes it to a `dict` (using
        `json.loads`).

        .. note::
           Decoding as XML is not supported.
        """
        if not self.as_xml:
            # only parse json when it exists, else just return None
            if not raw_body or raw_body == ' ':
                return None
            else:
                return json.loads(raw_body)
        else:
            raise NotImplementedError('Decoding as XML is not supported.')

    def delete(self, url, params=None):
        """
        Executes an HTTP DELETE request for the given URL

        params should be a urllib.urlencoded string
        """
        if params:
            url = '?'.join([url, params])

        response, content = self.http.request(url, method="DELETE",
                                              headers=self.headers)
        return self.process(response, content)

    def get(self, url, data=None):
        """
        Executes an HTTP GET request for the given URL

        data should be a dictionary of url parameters
        """
        if data:
            params = urlencode(data)
            url = '?'.join([url, params])

        response, content = self.http.request(url, method="GET",
                                              headers=self.headers)
        return self.process(response, content)

    def post(self, url, body=None):
        """
        Executes an HTTP POST request for the given URL
        """
        headers = self.headers
        headers['Content-Length'] = self.content_length(body)
        response, content = self.http.request(url, method="POST",
                                              body=body,
                                              headers=self.headers)

        return self.process(response, content)

    def put(self, url, data=None, body=None):
        """
        Executes an HTTP PUT request for the given URL
        """
        headers = self.headers
        headers['Content-Length'] = self.content_length(body)

        if data:
            params = urlencode(data)
            url = '?'.join([url, params])

        response, content = self.http.request(url, method="PUT",
                                              body=body,
                                              headers=headers)

        return self.process(response, content)

    def process(self, http_response, content):
        """
        Returns HTTP backend agnostic Response data
        """

        try:
            code = http_response.status
            body = self.decode(content)
            response = Response(code, body, content, http_response)

            return response

        except ValueError:
            raise ZencoderResponseError(http_response, content)

class Zencoder(object):
    """ This is the entry point to the Zencoder API """
    def __init__(self, api_key=None, api_version=None, as_xml=False, timeout=None, test=False):
        """
        Initializes Zencoder. You must have a valid API_KEY.

        You can pass in the api_key as an argument, or set
        `ZENCODER_API_KEY` as an environment variable, and it will use
        that, if api_key is unspecified.

        Set api_version='edge' to get the Zencoder development API. (defaults to 'v2')
        Set as_xml=True to get back xml data instead of the default json.
        """
        if not api_version:
            api_version = 'v2'

        self.base_url = 'https://app.zencoder.com/api/'
        if not api_version == 'edge':
            self.base_url = self.base_url + '%s/' % api_version

        if not api_key:
            try:
                self.api_key = os.environ['ZENCODER_API_KEY']
            except KeyError:
                raise ZencoderError('ZENCODER_API_KEY not set')
        else:
            self.api_key = api_key

        self.test = test
        self.as_xml = as_xml

        args = (self.base_url, self.api_key, self.as_xml)
        kwargs = dict(timeout=timeout, test=self.test, version=api_version)
        self.job = Job(*args, **kwargs)
        self.account = Account(*args, **kwargs)
        self.output = Output(*args, **kwargs)
        self.report = None
        if api_version == 'v2':
            self.report = Report(*args, **kwargs)

class Response(object):
    """
    The Response object stores the details of an API request in an XML/JSON
    agnostic way.
    """
    def __init__(self, code, body, raw_body, raw_response):
        self.code = code
        self.body = body
        self.raw_body = raw_body
        self.raw_response = raw_response

class Account(HTTPBackend):
    """ Account object """
    def __init__(self, *args, **kwargs):
        """
        Initializes an Account object
        """
        kwargs['resource_name'] = 'account'
        super(Account, self).__init__(*args, **kwargs)

    def create(self, email, tos=1, options=None):
        """
        Creates an account with Zencoder, no API Key necessary.
        """
        data = {'email': email,
                'terms_of_service': str(tos)}
        if options:
            data.update(options)

        return self.post(self.base_url, body=self.encode(data))

    def details(self):
        """
        Gets your account details.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url, data=data)

    def integration(self):
        """
        Puts your account into integration mode.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url + '/integration', data=data)

    def live(self):
        """
        Puts your account into live mode.
        """
        data = {'api_key': self.api_key}

        return self.get(self.base_url + '/live', data=data)

class Output(HTTPBackend):
    """ Gets information regarding outputs """
    def __init__(self, *args, **kwargs):
        """
        Contains all API methods relating to Outputs.
        """
        kwargs['resource_name'] = 'outputs'
        super(Output, self).__init__(*args, **kwargs)

    def progress(self, output_id):
        """
        Gets the given output id's progress.
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s/progress' % str(output_id),
                        data=data)

    def details(self, output_id):
        """
        Gets the given output id's details
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s' % str(output_id),
                        data=data)

class Job(HTTPBackend):
    """
    Contains all API methods relating to transcoding Jobs.
    """
    def __init__(self, *args, **kwargs):
        """
        Initializes a job object
        """
        kwargs['resource_name'] = 'jobs'
        super(Job, self).__init__(*args, **kwargs)

    def create(self, input, outputs=None, options=None):
        """
        Creates a job

        @param input: the input url as string
        @param outputs: a list of output dictionaries
        @param options: a dictionary of job options
        """
        as_test = int(self.test)

        data = {"api_key": self.api_key, "input": input, "test": as_test}
        if outputs:
            data['outputs'] = outputs

        if options:
            data.update(options)

        return self.post(self.base_url, body=self.encode(data))

    def list(self, page=1, per_page=50):
        """
        Lists some jobs.

        @param page: <int> the page of results to return
        @param per_page: <int> the number of results per page
        """
        data = {"api_key": self.api_key,
                "page": page,
                "per_page": per_page}
        return self.get(self.base_url, data=data)

    def details(self, job_id):
        """
        Gets details for the given job
        """
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s' % str(job_id), data=data)

    def progress(self, job_id):
        data = {'api_key': self.api_key}
        return self.get(self.base_url + '/%s/progress' % str(job_id), data=data)

    def resubmit(self, job_id):
        """
        Resubmits the given `job_id`
        """
        data = {'api_key': self.api_key}
        url = self.base_url + '/%s/resubmit' % str(job_id)
        return self.put(url, data=data)

    def cancel(self, job_id):
        """
        Cancels the given `job_id`
        """
        if self.version == 'v1':
            verb = self.get
        else:
            verb = self.put

        data = {'api_key': self.api_key}
        url = self.base_url + '/%s/cancel' % str(job_id)
        return verb(url, data=data)

    def delete(self, job_id):
        """
        Deletes the given `job_id`

        WARNING: This method is aliased to `Job.cancel` -- it is deprecated in
                 API version 2 and greater.
        """
        return self.cancel(job_id)

class Report(HTTPBackend):
    def __init__(self, *args, **kwargs):
        """
        Contains all API methods relating to Outputs.
        """
        kwargs['resource_name'] = 'reports'
        super(Report, self).__init__(*args, **kwargs)

    def details(self, start_date=None, end_date=None, grouping=None):
        """
        Gets a detailed Report
        """
        data = {'api_key': self.api_key}
        if start_date:
            data['from'] = start_date

        if end_date:
            data['to'] = end_date

        if grouping:
            data['grouping'] = grouping

        url = self.base_url + '/minutes'
        return self.get(url, data=data)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

class _ApiVersion1(object):
    def __init__(self, api_key, session_id=None):
        self.api_key = api_key
        print("{}, {}".format(api_key, session_id))

    def zed_index(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def area_value_graphs(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_rich_list(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def average_area_sold_price(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def area_zed_indices(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def zoopla_estimates(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def average_sold_prices(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_listings(self, **kwargs):
        validate_query_arguments(kwargs)
        raise NotImplementedError("This method isn't yet implemented.")

    def get_session_id(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def refine_estimate(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def arrange_viewing(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def local_info_graphs(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_historic_listings(self):
        raise NotImplementedError("This method isn't yet implemented.")

def validate_query_arguments(arguments):
    validated = {}
    for argument, value in arguments.items():
        validated[argument] = validate_argument(argument, value)

def validate_argument(name, value):
    try:
        validate_func = globals()['validate_' + name]
    except KeyError:
        return value
    else:
        return validate_func(value)

def validate_area(area):
    return True

def validate_lat_min(value):
    float(value)
    return value
<commit_msg>Added download_url and install_cache functions<commit_after>#!/usr/bin/env python
# encoding: utf-8

import requests
import requests_cache
from cStringIO import StringIO

class _ApiVersion1(object):
    def __init__(self, api_key, session_id=None):
        self.api_key = api_key
        print("{}, {}".format(api_key, session_id))

    def zed_index(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def area_value_graphs(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_rich_list(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def average_area_sold_price(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def area_zed_indices(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def zoopla_estimates(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def average_sold_prices(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_listings(self, **kwargs):
        validate_query_arguments(kwargs)
        raise NotImplementedError("This method isn't yet implemented.")

    def get_session_id(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def refine_estimate(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def arrange_viewing(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def local_info_graphs(self):
        raise NotImplementedError("This method isn't yet implemented.")

    def property_historic_listings(self):
        raise NotImplementedError("This method isn't yet implemented.")

def install_cache(expire_after):
    requests_cache.install_cache(
        expire_after=expire_after,
        allowable_methods=('GET',))

def download_url(url):
    response = requests.get(url)
    response.raise_for_status()
    return StringIO(response.content)

def validate_query_arguments(arguments):
    validated = {}
    for argument, value in arguments.items():
        validated[argument] = validate_argument(argument, value)

def validate_argument(name, value):
    try:
        validate_func = globals()['validate_' + name]
    except KeyError:
        return value
    else:
        return validate_func(value)

def validate_area(area):
    return True

def validate_lat_min(value):
    float(value)
    return value
<|endoftext|>


<commit_before>#!/usr/bin/env python

# relative_rv.py

# Script to calculate the relative Radial Velocity between two given wavelengths

# Jason Neal
# December 2016
import argparse

def relative_rv(wav_1, wav_2):
    """ Calculate the radial velocity difference between two wavelength values"""
    c = 299792.458   #  km / s
    difference = wav_2 - wav_1
    relative = difference / wav_1
    return relative * c

def _parser():
    """Take care of all the argparse stuff.

    :returns: the args
    """
    parser = argparse.ArgumentParser(description='Calculate RV doppler shift between two wavelengths')
    parser.add_argument('original_wavelength', help='The original wavelength', type=float)
    parser.add_argument('new_wavelength', help='THe new wavelength', type=float)
    parser.add_argument('-m', '--mode', help='Output mode', type=str)   # reverse, both, (iteratorable input)
    # TODO: Add these extra modes
    # parser.add_argument('-u', '--unit', help='Distance unit of measurement [km, m, cm]')
    # parser.add_argument('-r', '--round', help='Rounding on ouput, e.g. ["2dp", "3sf"]')
    args = parser.parse_args()
    return args

def main(original_wavelength, new_wavelength, mode=False):  # unit, rounding
    """ Obtian RV offset between wavelengths and print to screen.

    prints rv between the wavelength values.
    """
    rv = relative_rv(original_wavelength, new_wavelength)
    print("\n" + "-" * 20 )
    print("Original   -->   New   |  RV (km/s)" )
    print("-" * 20 )
    print("{0}  -->  {1}  |  {2}".format(original_wavelength, new_wavelength, rv))
    print("-" * 20 )

if __name__ == "__main__":
    args = vars(_parser())
    wave1 = args.pop('original_wavelength')
    wave2 = args.pop('new_wavelength')
    opts = {k: args[k] for k in args}

    main(wave1, wave2, **opts)
<commit_msg>Add a reverse mode<commit_after>#!/usr/bin/env python

# relative_rv.py

# Script to calculate the relative Radial Velocity between two given wavelengths

# Jason Neal
# December 2016
import argparse

def relative_rv(wav_1, wav_2):
    """ Calculate the radial velocity difference between two wavelength values"""
    c = 299792.458   #  km / s
    difference = wav_2 - wav_1
    relative = difference / wav_1
    return relative * c

def _parser():
    """Take care of all the argparse stuff.

    :returns: the args
    """
    parser = argparse.ArgumentParser(description='Calculate RV doppler shift between two wavelengths')
    parser.add_argument('original_wavelength', help='The original wavelength', type=float)
    parser.add_argument('new_wavelength', help='THe new wavelength', type=float)
    parser.add_argument('-m', '--mode', choices=['normal','reverse', 'iterable'], default='normal', help='Output mode', type=str)   # reverse, both, (iteratorable input)
    # TODO: Add these extra modes
    # parser.add_argument('-u', '--unit', help='Distance unit of measurement [km, m, cm]')
    # parser.add_argument('-r', '--round', help='Rounding on ouput, e.g. ["2dp", "3sf"]')
    args = parser.parse_args()
    return args

def main(original_wavelength, new_wavelength, mode=False):  # unit, rounding
    """ Obtian RV offset between wavelengths and print to screen.

    prints rv between the wavelength values.
    """
    if mode == "reverse":
        # Switch direction of doppler shift
        original_wavelength, new_wavelength = new_wavelength, original_wavelength

    rv = relative_rv(original_wavelength, new_wavelength)
    print("\n" + "-" * 20 )
    print("Original   -->   New   |  RV (km/s)" )
    print("-" * 20 )
    print("{0}  -->  {1}  |  {2}".format(original_wavelength, new_wavelength, rv))
    print("-" * 20 )

if __name__ == "__main__":
    args = vars(_parser())
    wave1 = args.pop('original_wavelength')
    wave2 = args.pop('new_wavelength')
    opts = {k: args[k] for k in args}

    main(wave1, wave2, **opts)
<|endoftext|>


<commit_before>import logger
import logging
import os
import re
import yaml
import shutil

DEFAULT_CONFIG_FILE = 'config.yml'
DEFAULT_VALIDATE_BEFORE = True
DEFAULT_VALIDATE_AFTER = False
DEFAULT_MUST_INCLUDE = []

repex_lgr = logger.init()
verbose_output = False

def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the repex_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        repex_lgr.setLevel(logging.DEBUG)
    else:
        repex_lgr.setLevel(logging.INFO)
    # print 'level is: ' + str(repex_lgr.getEffectiveLevel())

def import_config(config_file):
    """returns a configuration object

    :param string config_file: path to config file
    """
    # get config file path
    repex_lgr.debug('config file is: {}'.format(config_file))
    # append to path for importing
    try:
        repex_lgr.debug('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        repex_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        repex_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')
    # except SyntaxError:
    #     repex_lgr.error('config file syntax is malformatted. please fix '
    #                     'any syntax errors you might have and try again.')
    #     raise RuntimeError('bad config file')

def get_all_files(ftype, path, base_dir):
    repex_lgr.info('looking for {0}\'s under {1} in {2}'.format(
        ftype, path, base_dir))
    dirs = []
    for obj in os.listdir(base_dir):
        lookup_dir = os.path.join(base_dir, obj)
        # repex_lgr.info('looking for {0} in lookup dir: {1}'.format(
        #    path, lookup_dir))
        if os.path.isdir(lookup_dir) and re.search(
                r'{0}'.format(path), lookup_dir):
            dirs.append(obj)

    # dirs = [
    #     x for x in os.listdir(base_dir)
    #     if os.path.isdir(lookup_dir) and re.search(
    #         r'{0}'.format(path), lookup_dir)
    # ]
    # repex_lgr.info('dirs: {0}'.format(dirs))
    target_files = []
    for directory in dirs:
        # repex_lgr.info('iterating over {0}'.format(
        #    os.path.join(base_dir, directory)))
        for root, dirs, files in os.walk(os.path.join(base_dir, directory)):
            for f in files:
                if f == ftype:
                    # repex_lgr.info('found file:' + f)
                    target_files.append(os.path.join(root, f))
    # repex_lgr.info('files: {0}'.format(target_files))
    return target_files

def iterate(configfile, variables=None, verbose=False):
    """iterates over all files in `configfile`

    :param string configfile: yaml path with files to iterate over
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    config = import_config(configfile)
    try:
        paths = config['paths']
    except TypeError:
        raise RepexError('no paths configured')

    for path in paths:
        handle_path(path, variables, verbose=False)

def handle_path(p, variables, verbose=False):
    _set_global_verbosity_level(verbose)
    if os.path.isfile(p['path']):
        if p.get('base_directory'):
            repex_lgr.info(
                'base_directory is irrelevant when dealing with single files')
        handle_file(p, variables, verbose)
    else:
        if p.get('to_file'):
            raise RepexError(
                '"to_file" requires explicit "path"')
        files = get_all_files(p['type'], p['path'], p['base_directory'])
        repex_lgr.info(files)
        for f in files:
            p['path'] = f
            handle_file(p, variables, verbose)

def handle_file(f, variables=None, verbose=False):
    """handle a single file

    this will perform a validation if necessary and then
    perform the replacement in the file.

    :param dict file: a dict of a single file's properties
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    variables = variables if variables else {}
    if type(variables) is not dict:
        raise RuntimeError('variables must be of type dict')
    p = Repex(
        f['path'],
        f['match'],
        f['replace'],
        f['with'],
        f.get('to_file', False),
        verbose
    )
    validate_before = f.get('validate_before', DEFAULT_VALIDATE_BEFORE)
    must_include = f.get('must_include', DEFAULT_MUST_INCLUDE)
    if validate_before and not p.validate_before(must_include):
            raise RepexError('prevalidation failed')
    matches = p.find_matches()
    p.extend_vars(variables)
    p.replace(matches)

class Repex():
    def __init__(self, path, match, pattern, rwith,
                 to_file=False, verbose=False):
        self.path = path
        self.match = match
        self.pattern = pattern
        self.rwith = rwith
        self.to_file = to_file
        _set_global_verbosity_level(verbose)

    def validate_before(self, must_include=[]):

        def verify_includes(must_include):
            """verifies that all required strings are in the file
            """
            # first, see if the pattern is even in the file.
            repex_lgr.debug('looking for required strings')
            repex_lgr.debug('must include is {0}'.format(must_include))

            # iterate over the strings and verify that
            # they exist in the file
            included = True
            for string in must_include:
                with open(self.path) as f:
                    if not any(re.search(r'{0}'.format(
                            string), line) for line in f):
                        repex_lgr.error(
                            'required string {0} not found in {1}'.format(
                                string, self.path))
                        included = False
            if not included:
                return False
            return True

        def validate_pattern():
            """verifies that the pattern exists
            """
            repex_lgr.debug('looking for pattern to replace')
            # verify that the pattern you're looking to replace
            # exists in the file
            with open(self.path) as f:
                content = f.read()
                repex_lgr.info('file content is: {0}'.format(content))
                repex_lgr.info('file is: {0}'.format(self.path))
                if not re.search(r'{0}'.format(self.match), content):
                    repex_lgr.warning('match {0} not found in {1}'.format(
                        self.match, self.path))
                    return False
                else:
                    m = re.findall(self.match, content)
                    repex_lgr.info('matches are: {0}'.format(m))
                    if not any(re.search(r'{0}'.format(
                            self.pattern), match) for match in m):
                        # pattern does not occur in file so we are done.
                        repex_lgr.warning(
                            'pattern {0} not found in any matches'.format(
                                self.pattern))
                        return False
                    repex_lgr.debug(
                        'pattern {0} found in one or more matches'.format(
                            self.pattern))
                return True

        if must_include:
            return verify_includes(must_include)
        return validate_pattern()

    def find_matches(self):
        """finds all matches of an expression in a file
        """
        with open(self.path) as f:
            return re.findall(self.match, f.read())

    def extend_vars(self, v):
        """extends the variables to their corresponding values
        """
        # TODO: (IMPRV) replace with dict for a more implicit
        # TODO: (IMPRV) implementation
        # iterate over all variables
        for var, value in v.items():
            # replace variable in pattern
            self.pattern = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.pattern)
            # replace variable in input data
            self.rwith = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.rwith)
            self.match = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.match)
            self.path = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.path)

    def replace(self, matches):
        """replaces all occurences of the regex in all matches
        from a file with a specific value.
        """
        temp_file = self.path + ".tmp"
        repex_lgr.info('{0}: replacing {1} with {2}'.format(
            self.path, self.pattern, self.rwith))
        repex_lgr.info('matches are: {0}'.format(matches))
        with open(self.path) as f:
            content = f.read()
        for m in matches:
            # replace pattern in match
            r = re.sub(self.pattern, self.rwith, m)
            repex_lgr.info('replacing {0} with {1}'.format(m, r))
            # then replace the previous match with the newly formatted one
            content = content.replace(m, r)
            repex_lgr.info('content: {0}'.format(content))
        with open(temp_file, "w") as out:
            out.write(content)
        output_file = self.to_file if self.to_file else self.path
        repex_lgr.info('writing output to {0}'.format(output_file))
        shutil.move(temp_file, output_file)

class RepexError(Exception):
    pass
<commit_msg>removed content printing<commit_after>import logger
import logging
import os
import re
import yaml
import shutil

DEFAULT_CONFIG_FILE = 'config.yml'
DEFAULT_VALIDATE_BEFORE = True
DEFAULT_VALIDATE_AFTER = False
DEFAULT_MUST_INCLUDE = []

repex_lgr = logger.init()
verbose_output = False

def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the repex_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        repex_lgr.setLevel(logging.DEBUG)
    else:
        repex_lgr.setLevel(logging.INFO)
    # print 'level is: ' + str(repex_lgr.getEffectiveLevel())

def import_config(config_file):
    """returns a configuration object

    :param string config_file: path to config file
    """
    # get config file path
    repex_lgr.debug('config file is: {}'.format(config_file))
    # append to path for importing
    try:
        repex_lgr.debug('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        repex_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        repex_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')
    # except SyntaxError:
    #     repex_lgr.error('config file syntax is malformatted. please fix '
    #                     'any syntax errors you might have and try again.')
    #     raise RuntimeError('bad config file')

def get_all_files(ftype, path, base_dir):
    repex_lgr.info('looking for {0}\'s under {1} in {2}'.format(
        ftype, path, base_dir))
    dirs = []
    for obj in os.listdir(base_dir):
        lookup_dir = os.path.join(base_dir, obj)
        # repex_lgr.info('looking for {0} in lookup dir: {1}'.format(
        #    path, lookup_dir))
        if os.path.isdir(lookup_dir) and re.search(
                r'{0}'.format(path), lookup_dir):
            dirs.append(obj)

    # dirs = [
    #     x for x in os.listdir(base_dir)
    #     if os.path.isdir(lookup_dir) and re.search(
    #         r'{0}'.format(path), lookup_dir)
    # ]
    # repex_lgr.info('dirs: {0}'.format(dirs))
    target_files = []
    for directory in dirs:
        # repex_lgr.info('iterating over {0}'.format(
        #    os.path.join(base_dir, directory)))
        for root, dirs, files in os.walk(os.path.join(base_dir, directory)):
            for f in files:
                if f == ftype:
                    # repex_lgr.info('found file:' + f)
                    target_files.append(os.path.join(root, f))
    # repex_lgr.info('files: {0}'.format(target_files))
    return target_files

def iterate(configfile, variables=None, verbose=False):
    """iterates over all files in `configfile`

    :param string configfile: yaml path with files to iterate over
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    config = import_config(configfile)
    try:
        paths = config['paths']
    except TypeError:
        raise RepexError('no paths configured')

    for path in paths:
        handle_path(path, variables, verbose=False)

def handle_path(p, variables, verbose=False):
    _set_global_verbosity_level(verbose)
    if os.path.isfile(p['path']):
        if p.get('base_directory'):
            repex_lgr.info(
                'base_directory is irrelevant when dealing with single files')
        handle_file(p, variables, verbose)
    else:
        if p.get('to_file'):
            raise RepexError(
                '"to_file" requires explicit "path"')
        files = get_all_files(p['type'], p['path'], p['base_directory'])
        repex_lgr.info(files)
        for f in files:
            p['path'] = f
            handle_file(p, variables, verbose)

def handle_file(f, variables=None, verbose=False):
    """handle a single file

    this will perform a validation if necessary and then
    perform the replacement in the file.

    :param dict file: a dict of a single file's properties
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    variables = variables if variables else {}
    if type(variables) is not dict:
        raise RuntimeError('variables must be of type dict')
    p = Repex(
        f['path'],
        f['match'],
        f['replace'],
        f['with'],
        f.get('to_file', False),
        verbose
    )
    validate_before = f.get('validate_before', DEFAULT_VALIDATE_BEFORE)
    must_include = f.get('must_include', DEFAULT_MUST_INCLUDE)
    if validate_before and not p.validate_before(must_include):
            raise RepexError('prevalidation failed')
    matches = p.find_matches()
    p.extend_vars(variables)
    p.replace(matches)

class Repex():
    def __init__(self, path, match, pattern, rwith,
                 to_file=False, verbose=False):
        self.path = path
        self.match = match
        self.pattern = pattern
        self.rwith = rwith
        self.to_file = to_file
        _set_global_verbosity_level(verbose)

    def validate_before(self, must_include=[]):

        def verify_includes(must_include):
            """verifies that all required strings are in the file
            """
            # first, see if the pattern is even in the file.
            repex_lgr.debug('looking for required strings')
            repex_lgr.debug('must include is {0}'.format(must_include))

            # iterate over the strings and verify that
            # they exist in the file
            included = True
            for string in must_include:
                with open(self.path) as f:
                    if not any(re.search(r'{0}'.format(
                            string), line) for line in f):
                        repex_lgr.error(
                            'required string {0} not found in {1}'.format(
                                string, self.path))
                        included = False
            if not included:
                return False
            return True

        def validate_pattern():
            """verifies that the pattern exists
            """
            repex_lgr.debug('looking for pattern to replace')
            # verify that the pattern you're looking to replace
            # exists in the file
            with open(self.path) as f:
                content = f.read()
                repex_lgr.info('file content is: {0}'.format(content))
                repex_lgr.info('file is: {0}'.format(self.path))
                if not re.search(r'{0}'.format(self.match), content):
                    repex_lgr.warning('match {0} not found in {1}'.format(
                        self.match, self.path))
                    return False
                else:
                    m = re.findall(self.match, content)
                    repex_lgr.info('matches are: {0}'.format(m))
                    if not any(re.search(r'{0}'.format(
                            self.pattern), match) for match in m):
                        # pattern does not occur in file so we are done.
                        repex_lgr.warning(
                            'pattern {0} not found in any matches'.format(
                                self.pattern))
                        return False
                    repex_lgr.debug(
                        'pattern {0} found in one or more matches'.format(
                            self.pattern))
                return True

        if must_include:
            return verify_includes(must_include)
        return validate_pattern()

    def find_matches(self):
        """finds all matches of an expression in a file
        """
        with open(self.path) as f:
            return re.findall(self.match, f.read())

    def extend_vars(self, v):
        """extends the variables to their corresponding values
        """
        # TODO: (IMPRV) replace with dict for a more implicit
        # TODO: (IMPRV) implementation
        # iterate over all variables
        for var, value in v.items():
            # replace variable in pattern
            self.pattern = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.pattern)
            # replace variable in input data
            self.rwith = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.rwith)
            self.match = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.match)
            self.path = re.sub("{{ " + ".{0}".format(
                var) + " }}", str(v[var]), self.path)

    def replace(self, matches):
        """replaces all occurences of the regex in all matches
        from a file with a specific value.
        """
        temp_file = self.path + ".tmp"
        repex_lgr.info('{0}: replacing {1} with {2}'.format(
            self.path, self.pattern, self.rwith))
        repex_lgr.debug('matches are: {0}'.format(matches))
        with open(self.path) as f:
            content = f.read()
        for m in matches:
            # replace pattern in match
            r = re.sub(self.pattern, self.rwith, m)
            repex_lgr.info('replacing {0} with {1}'.format(m, r))
            # then replace the previous match with the newly formatted one
            content = content.replace(m, r)
        with open(temp_file, "w") as out:
            out.write(content)
        output_file = self.to_file if self.to_file else self.path
        repex_lgr.info('writing output to {0}'.format(output_file))
        shutil.move(temp_file, output_file)

class RepexError(Exception):
    pass
<|endoftext|>


<commit_before>import logger
import logging
import os
import re
import yaml
import shutil

DEFAULT_CONFIG_FILE = 'config.yml'
DEFAULT_VALIDATE_BEFORE = True
DEFAULT_VALIDATE_AFTER = False
DEFAULT_MUST_INCLUDE = []

repex_lgr = logger.init()
verbose_output = False

def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the repex_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        repex_lgr.setLevel(logging.DEBUG)
    else:
        repex_lgr.setLevel(logging.INFO)
    # print 'level is: ' + str(repex_lgr.getEffectiveLevel())

def import_config(config_file):
    """returns a configuration object

    :param string config_file: path to config file
    """
    # get config file path
    repex_lgr.debug('config file is: {}'.format(config_file))
    # append to path for importing
    try:
        repex_lgr.info('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        repex_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        repex_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')

def get_all_files(file_name_regex, path, base_dir, excluded_paths=None,
                  verbose=False, exclude_file_name_regex=None):
    _set_global_verbosity_level(verbose)
    excluded_paths = excluded_paths if excluded_paths else []
    repex_lgr.debug('excluded paths: {0}'.format(excluded_paths))
    if type(excluded_paths) is not list:
        raise RepexError(
            'excluded_paths must be of type list (not {0})'.format(
                type(excluded_paths)))
    repex_lgr.info('looking for {0}\'s under {1} in {2}'.format(
        file_name_regex, path, base_dir))
    dirs = []
    for obj in os.listdir(base_dir):
        lookup_dir = os.path.join(base_dir, obj)
        if os.path.isdir(lookup_dir) and re.search(
                r'{0}'.format(path), lookup_dir):
            if lookup_dir in excluded_paths:
                repex_lgr.info('path {0} is excluded, skipping.'.format(obj))
            else:
                dirs.append(obj)
    target_files = []
    for directory in dirs:
        for root, dirs, files in os.walk(os.path.join(base_dir, directory)):
            # append base dir to excluded paths to receive the full path
            # relative to the base_dir.
            ex_paths = [os.path.join(base_dir, e) for e in excluded_paths]
            if root in ex_paths:
                repex_lgr.info('path {0} is excluded, skipping.'.format(root))
                continue
            for f in files:
                if re.match(
                        r'{0}'.format(file_name_regex), f)\
                        and not re.match(
                            r'{0}'.format(exclude_file_name_regex), f)\
                        and not os.path.join(root, f) in ex_paths:
                    target_files.append(os.path.join(root, f))
                elif os.path.join(root, f) in ex_paths:
                    repex_lgr.debug('path {0} is excluded, skipping.'.format(
                        os.path.join(root, f)))
    return target_files

class VarHandler():

    def __init__(self, path_attributes):
        self.attributes = path_attributes

    def expand(self, v):
        """expands the variables to their corresponding values
        """
        # TODO: (IMPRV) replace with dict for a more implicit
        # TODO: (IMPRV) implementation
        # TODO: (IMPRV) add expansion tests
        # iterate over all variables
        repex_lgr.info('expanding variables...')
        for var, value in v.items():
            for attribute in self.attributes.keys():
                if isinstance(self.attributes[attribute], str):
                    self.attributes[attribute] = \
                        self.expand_var(
                            var, v[var], self.attributes[attribute])
        return self.attributes

    def expand_var(self, variable, value, in_string):

        def check_if_expanded(string, variable):
            repex_lgr.debug('verifying that string {0} expanded'.format(
                string))
            if re.search('{{ ' + '.{0}'.format(variable) + ' }}', string):
                repex_lgr.error('string {0} failed to expand'.format(string))
                raise RepexError('string failed to expand')

        var = "{{ " + ".{0}".format(variable) + " }}"
        if re.search(var, in_string):
            repex_lgr.debug('expanding var {0} to {1} in {2}'.format(
                variable, value, in_string))
            expanded_variable = re.sub("{{ " + ".{0}".format(
                variable) + " }}", str(value), in_string)
            check_if_expanded(expanded_variable, variable)
            return expanded_variable
        return in_string

def iterate(configfile, variables=None, verbose=False):
    """iterates over all files in `configfile`

    :param string configfile: yaml path with files to iterate over
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    config = import_config(configfile)
    try:
        paths = config['paths']
    except TypeError:
        raise RepexError('no paths configured')

    for path in paths:
        handle_path(path, variables, verbose)

def handle_path(p, variables, verbose=False):
    _set_global_verbosity_level(verbose)
    variables = variables if variables else {}
    if type(variables) is not dict:
        raise RuntimeError('variables must be of type dict')
    var_expander = VarHandler(p)
    p = var_expander.expand(variables)
    if os.path.isfile(p['path']):
        if p.get('base_directory'):
            repex_lgr.info(
                'base_directory is irrelevant when dealing with single files')
        handle_file(p, variables, verbose)
    else:
        if p.get('to_file'):
            raise RepexError(
                '"to_file" requires explicit "path"')
        files = get_all_files(
            p['type'], p['path'], p['base_directory'], p['excluded'], verbose)
        repex_lgr.info('files found: {0}'.format(files))
        for f in files:
            p['path'] = f
            handle_file(p, variables, verbose)

def handle_file(f, variables=None, verbose=False):
    """handle a single file

    this will perform a validation if necessary and then
    perform the replacement in the file.

    :param dict file: a dict of a single file's properties
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    p = Repex(
        f['path'],
        f['match'],
        f['replace'],
        f['with'],
        f.get('to_file', False),
        verbose
    )
    repex_lgr.debug('vars: {0}'.format(variables))
    validate_before = f.get('validate_before', DEFAULT_VALIDATE_BEFORE)
    if not isinstance(validate_before, bool):
        raise RepexError('validate_before must be either of type boolean')
    must_include = f.get('must_include', DEFAULT_MUST_INCLUDE)
    if validate_before and not p.validate_before(must_include=must_include):
        raise RepexError('prevalidation failed')
    matches = p.find_matches()
    p.replace(matches)

class Repex():
    def __init__(self, path, match, pattern, rwith,
                 to_file=False, verbose=False):
        self.path = path
        self.match = match
        self.pattern = pattern
        self.rwith = rwith
        self.to_file = to_file
        with open(self.path) as f:
            self.content = f.read()
        _set_global_verbosity_level(verbose)

    def validate_before(self, force_match=False, force_pattern=False,
                        must_include=[]):

        def verify_includes(must_include):
            """verifies that all required strings are in the file
            """
            repex_lgr.debug('looking for required strings: {0}'.format(
                must_include))
            # iterate over the strings and verify that
            # they exist in the file
            included = True
            for string in must_include:
                with open(self.path) as f:
                    if not any(re.search(r'{0}'.format(
                            string), line) for line in f):
                        repex_lgr.error(
                            'required string "{0}" not found in {1}'.format(
                                string, self.path))
                        included = False
            if not included:
                return False
            repex_lgr.debug('required strings found')
            return True

        def validate_pattern(force_pattern, force_match):
            """verifies that the pattern exists
            """
            repex_lgr.debug('looking for pattern {0}'.format(self.pattern))
            # verify that the pattern you're looking to replace
            # exists in the file
            if not re.search(self.match, self.content):
                repex_lgr.warning('match {0} not found in {1}'.format(
                    self.match, self.path))
                if force_match:
                    return False
                return True
            else:
                # find all matches of the pattern in the file
                m = self.find_matches()
                if not any(re.search(r'{0}'.format(
                        self.pattern), match) for match in m):
                    # pattern does not occur in file so we are done.
                    repex_lgr.warning(
                        'pattern {0} not found in any matches'.format(
                            self.pattern))
                    if force_pattern:
                        return False
                    return True
            repex_lgr.debug('pattern found in one or more matches')
            return True

        if must_include and not verify_includes(must_include) \
                or not validate_pattern(force_pattern, force_match):
            return False
        return True

    def find_matches(self):
        """finds all matches of an expression in a file
        """
        r = re.compile('(?P<matchgroup>{0})'.format(
            self.match))
        x = [match.groupdict() for match in r.finditer(self.content)]
        m = [d['matchgroup'] for d in x if d.get('matchgroup')]
        repex_lgr.debug('matches found: {0}'.format(m))
        return m

    def replace(self, matches):
        """replaces all occurences of the regex in all matches
        from a file with a specific value.
        """
        temp_file = self.path + ".tmp"
        repex_lgr.debug('matches to replace: {0}'.format(matches))
        with open(self.path) as f:
            content = f.read()
        for m in matches:
            # replace pattern in match
            r = re.sub(self.pattern, self.rwith, m)
            repex_lgr.info('replacing {0} with {1} in {2}'.format(
                m, r, self.path))
            # then replace the previous match with the newly formatted one
            content = content.replace(m, r)
        with open(temp_file, "w") as out:
            out.write(content)
        output_file = self.to_file if self.to_file else self.path
        repex_lgr.info('writing output to {0}'.format(output_file))
        shutil.move(temp_file, output_file)

class RepexError(Exception):
    pass
<commit_msg>some more documentation<commit_after>import logger
import logging
import os
import re
import yaml
import shutil

DEFAULT_CONFIG_FILE = 'config.yml'
DEFAULT_VALIDATE_BEFORE = True
DEFAULT_VALIDATE_AFTER = False
DEFAULT_MUST_INCLUDE = []

repex_lgr = logger.init()
verbose_output = False

def _set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the repex_lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
    # TODO: (IMPRV) only raise exceptions in verbose mode
    verbose_output = is_verbose_output
    if verbose_output:
        repex_lgr.setLevel(logging.DEBUG)
    else:
        repex_lgr.setLevel(logging.INFO)
    # print 'level is: ' + str(repex_lgr.getEffectiveLevel())

def import_config(config_file):
    """returns a configuration object

    :param string config_file: path to config file
    """
    # get config file path
    repex_lgr.debug('config file is: {}'.format(config_file))
    # append to path for importing
    try:
        repex_lgr.info('importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())
    except IOError as ex:
        repex_lgr.error(str(ex))
        raise RuntimeError('cannot access config file')
    except yaml.parser.ParserError as ex:
        repex_lgr.error('invalid yaml file: {0}'.format(ex))
        raise RuntimeError('invalid yaml file')

def get_all_files(file_name_regex, path, base_dir, excluded_paths=None,
                  verbose=False, exclude_file_name_regex=None):
    _set_global_verbosity_level(verbose)
    excluded_paths = excluded_paths if excluded_paths else []
    repex_lgr.debug('excluded paths: {0}'.format(excluded_paths))
    if type(excluded_paths) is not list:
        raise RepexError(
            'excluded_paths must be of type list (not {0})'.format(
                type(excluded_paths)))
    repex_lgr.info('looking for {0}\'s under {1} in {2}'.format(
        file_name_regex, path, base_dir))
    dirs = []
    for obj in os.listdir(base_dir):
        lookup_dir = os.path.join(base_dir, obj)
        if os.path.isdir(lookup_dir) and re.search(
                r'{0}'.format(path), lookup_dir):
            if lookup_dir in excluded_paths:
                repex_lgr.info('path {0} is excluded, skipping.'.format(obj))
            else:
                dirs.append(obj)
    target_files = []
    for directory in dirs:
        for root, dirs, files in os.walk(os.path.join(base_dir, directory)):
            # append base dir to excluded paths to receive the full path
            # relative to the base_dir.
            ex_paths = [os.path.join(base_dir, e) for e in excluded_paths]
            if root in ex_paths:
                repex_lgr.info('path {0} is excluded, skipping.'.format(root))
                continue
            for f in files:
                if re.match(
                        r'{0}'.format(file_name_regex), f)\
                        and not re.match(
                            r'{0}'.format(exclude_file_name_regex), f)\
                        and not os.path.join(root, f) in ex_paths:
                    target_files.append(os.path.join(root, f))
                elif os.path.join(root, f) in ex_paths:
                    repex_lgr.debug('path {0} is excluded, skipping.'.format(
                        os.path.join(root, f)))
    return target_files

class VarHandler():
    """handles variable expansion and replacement
    """
    def __init__(self, verbose=False):
        _set_global_verbosity_level(verbose)

    def expand(self, vars, attributes):
        """receives a list of variables and a dict of attributes
        and iterates through them to expand a variable in an attribute

        attributes:

        type: VERSION
        path: resources
        excluded:
            - excluded_file.file
        base_directory: '{{ .base_dir }}'
        match: '"version": "\d+\.\d+(\.\d+)?(-\w\d+)?'
        replace: \d+\.\d+(\.\d+)?(-\w\d+)?
        with: "{{ .version }}"
        validate_before: true
        must_include:
            - date
            - commit
            - version

        variables:

        {
            'version': 3,
            'base_dir': .
        }

        Will only replace in attributes of type `string`. So, for instance,
        the values in `excluded`, `must_include` and `validate_before`
        will not be replaced.

        :param dict vars: dict of variables
        :param dict attributes: dict of attributes as shown above.
        """

        # iterate over all variables
        repex_lgr.info('expanding variables...')
        for var, value in vars.items():
            for attribute in attributes.keys():
                if isinstance(attributes[attribute], str):
                    attributes[attribute] = self.expand_var(
                        var, vars[var], attributes[attribute])
        return attributes

    def expand_var(self, variable, value, in_string):
        """expands variable to its corresponding value in_string

        :param string variable: variable name
        :param value: value to replace with
        :param string in_string: the string to replace in
        """
        def check_if_expanded(string, variable):
            repex_lgr.debug('verifying that string {0} expanded'.format(
                string))
            if re.search('{{ ' + '.{0}'.format(variable) + ' }}', string):
                repex_lgr.error('string {0} failed to expand'.format(string))
                raise RepexError('string failed to expand')

        var = "{{ " + ".{0}".format(variable) + " }}"
        if re.search(var, in_string):
            repex_lgr.debug('expanding var {0} to {1} in {2}'.format(
                variable, value, in_string))
            expanded_variable = re.sub("{{ " + ".{0}".format(
                variable) + " }}", str(value), in_string)
            check_if_expanded(expanded_variable, variable)
            return expanded_variable
        return in_string

def iterate(configfile, variables=None, verbose=False):
    """iterates over all files in `configfile`

    :param string configfile: yaml path with files to iterate over
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    config = import_config(configfile)
    try:
        paths = config['paths']
    except TypeError:
        raise RepexError('no paths configured')

    for path in paths:
        handle_path(path, variables, verbose)

def handle_path(p, variables, verbose=False):
    _set_global_verbosity_level(verbose)
    variables = variables if variables else {}
    if type(variables) is not dict:
        raise RuntimeError('variables must be of type dict')
    var_expander = VarHandler(verbose)
    p = var_expander.expand(variables, p)
    if os.path.isfile(p['path']):
        if p.get('base_directory'):
            repex_lgr.info(
                'base_directory is irrelevant when dealing with single files')
        handle_file(p, variables, verbose)
    else:
        if p.get('to_file'):
            raise RepexError(
                '"to_file" requires explicit "path"')
        files = get_all_files(
            p['type'], p['path'], p['base_directory'], p['excluded'], verbose)
        repex_lgr.info('files found: {0}'.format(files))
        for f in files:
            p['path'] = f
            handle_file(p, variables, verbose)

def handle_file(f, variables=None, verbose=False):
    """handle a single file

    this will perform a validation if necessary and then
    perform the replacement in the file.

    :param dict file: a dict of a single file's properties
    :param dict variables: a dict of variables (can be None)
    :param bool verbose: verbose output flag
    """
    _set_global_verbosity_level(verbose)
    p = Repex(
        f['path'],
        f['match'],
        f['replace'],
        f['with'],
        f.get('to_file', False),
        verbose
    )
    repex_lgr.debug('vars: {0}'.format(variables))
    validate_before = f.get('validate_before', DEFAULT_VALIDATE_BEFORE)
    if not isinstance(validate_before, bool):
        raise RepexError('validate_before must be either of type boolean')
    must_include = f.get('must_include', DEFAULT_MUST_INCLUDE)
    if validate_before and not p.validate_before(must_include=must_include):
        raise RepexError('prevalidation failed')
    matches = p.find_matches()
    p.replace(matches)

class Repex():
    def __init__(self, path, match, pattern, rwith,
                 to_file=False, verbose=False):
        self.path = path
        self.match = match
        self.pattern = pattern
        self.rwith = rwith
        self.to_file = to_file
        with open(self.path) as f:
            self.content = f.read()
        _set_global_verbosity_level(verbose)

    def validate_before(self, force_match=False, force_pattern=False,
                        must_include=[]):

        def verify_includes(must_include):
            """verifies that all required strings are in the file
            """
            repex_lgr.debug('looking for required strings: {0}'.format(
                must_include))
            # iterate over the strings and verify that
            # they exist in the file
            included = True
            for string in must_include:
                with open(self.path) as f:
                    if not any(re.search(r'{0}'.format(
                            string), line) for line in f):
                        repex_lgr.error(
                            'required string "{0}" not found in {1}'.format(
                                string, self.path))
                        included = False
            if not included:
                return False
            repex_lgr.debug('required strings found')
            return True

        def validate_pattern(force_pattern, force_match):
            """verifies that the pattern exists
            """
            repex_lgr.debug('looking for pattern {0}'.format(self.pattern))
            # verify that the pattern you're looking to replace
            # exists in the file
            if not re.search(self.match, self.content):
                repex_lgr.warning('match {0} not found in {1}'.format(
                    self.match, self.path))
                if force_match:
                    return False
                return True
            else:
                # find all matches of the pattern in the file
                m = self.find_matches()
                if not any(re.search(r'{0}'.format(
                        self.pattern), match) for match in m):
                    # pattern does not occur in file so we are done.
                    repex_lgr.warning(
                        'pattern {0} not found in any matches'.format(
                            self.pattern))
                    if force_pattern:
                        return False
                    return True
            repex_lgr.debug('pattern found in one or more matches')
            return True

        if must_include and not verify_includes(must_include) \
                or not validate_pattern(force_pattern, force_match):
            return False
        return True

    def find_matches(self):
        """finds all matches of an expression in a file
        """
        r = re.compile('(?P<matchgroup>{0})'.format(
            self.match))
        x = [match.groupdict() for match in r.finditer(self.content)]
        m = [d['matchgroup'] for d in x if d.get('matchgroup')]
        repex_lgr.debug('matches found: {0}'.format(m))
        return m

    def replace(self, matches):
        """replaces all occurences of the regex in all matches
        from a file with a specific value.
        """
        temp_file = self.path + ".tmp"
        repex_lgr.debug('matches to replace: {0}'.format(matches))
        with open(self.path) as f:
            content = f.read()
        for m in matches:
            # replace pattern in match
            r = re.sub(self.pattern, self.rwith, m)
            repex_lgr.info('replacing {0} with {1} in {2}'.format(
                m, r, self.path))
            # then replace the previous match with the newly formatted one
            content = content.replace(m, r)
        with open(temp_file, "w") as out:
            out.write(content)
        output_file = self.to_file if self.to_file else self.path
        repex_lgr.info('writing output to {0}'.format(output_file))
        shutil.move(temp_file, output_file)

class RepexError(Exception):
    pass
<|endoftext|>


<commit_before>import socket

client = socket.create_connection(("localhost", 8675))

while True:
	line = client.recv(1024)
	print line.replace("\n", "")<commit_msg>repl completed, testing for win<commit_after>import socket

client = socket.create_connection(("192.168.1.148", 8675)) # to connect back to home to $nc -l -p 8675

def reply(str):
	client.send(str + "\n")

while True: # <-----------------------+
	line = client.recv(1024) #read    |
	cmd = line.replace("\n", "") #    |
	print "==>", cmd #eval/exec       |
	exec(cmd) #                       |
	# print result #print               | # apparently no print
	# reply(result) #                   |
	# loop ---------------------------+<|endoftext|>


<commit_before># coding: utf-8
from browser import Browser
import pycurl
import simplejson

class RestClient(Browser):
    def __init__(self, base):
        super(RestClient, self).__init__()
        self._curl.setopt(pycurl.USERAGENT, "pycurl.rest_client 0.1")
        self.base = base

    def go(self, obj, uid=None):
        url = '%(base)s/%(obj)s' % {'base': self.base,
                                    'obj' : obj}
        if uid is not None:
            url += '/%s' % uid

        res = super(RestClient, self).go(url)
        assert res == 200, "Only handle success, not: %d" % res
        return res

    def get(self, obj, uid=None):
        self._curl.setopt(pycurl.HTTPGET, 1)
        self.go(obj, uid)
        return simplejson.loads(self.src)

    def head(self, obj, uid=None):
        self._curl.setopt(pycurl.HTTPGET, 1)
        self._curl.setopt(pycurl.NOBODY, 1)
        self.go(obj, uid)
        # TODO: care about headers

    def post(self, obj, data):
        self._curl.setopt(pycurl.POST, 1)
        self._curl.setopt(pycurl.POSTFIELDS, simplejson.loads(data))
        return simplejson.loads(self.src)

    def put(self, obj, uid, data):
        self._curl.setopt(pycurl.CUSTOMREQUEST, 'PUT')
        self._curl.setopt(pycurl.POSTFIELDS, simplejson.loads(data))
        return simplejson.loads(self.src)

    def delete(self, obj, uid):
        self._curl.setopt(pycurl.CUSTOMREQUEST, 'DELETE')
        # TODO: care about headers
<commit_msg>Factor out JSON<commit_after># coding: utf-8
from browser import Browser
import pycurl
import simplejson as json

class RestClient(Browser):

    """
    A simple REST client based upon pycurlbrowser
    """

    def __init__(self, base):
        super(RestClient, self).__init__()
        self._curl.setopt(pycurl.USERAGENT, "pycurl.rest_client 0.1")
        self.base = base

    def go(self, obj, uid=None):
        url = '%(base)s/%(obj)s' % {'base': self.base,
                                    'obj' : obj}
        if uid is not None:
            url += '/%s' % uid

        res = super(RestClient, self).go(url)
        assert res == 200, "Only handle success, not: %d" % res
        return res

    def get(self, obj, uid=None):
        self._curl.setopt(pycurl.HTTPGET, 1)
        self.go(obj, uid)
        return self.src

    def head(self, obj, uid=None):
        self._curl.setopt(pycurl.HTTPGET, 1)
        self._curl.setopt(pycurl.NOBODY, 1)
        self.go(obj, uid)
        # TODO: care about headers

    def post(self, obj, data=None):
        self._curl.setopt(pycurl.POST, 1)
        self._curl.setopt(pycurl.POSTFIELDS, data)
        self.go(obj)
        return self.src

    def put(self, obj, uid, data=None):
        self._curl.setopt(pycurl.CUSTOMREQUEST, 'PUT')
        self._curl.setopt(pycurl.POSTFIELDS, data)
        return self.src

    def delete(self, obj, uid):
        self._curl.setopt(pycurl.CUSTOMREQUEST, 'DELETE')
        # TODO: care about headers

class RestClientJson(RestClient):
    """
    A REST client that only speaks JSON
    """

    def get(self, obj, uid=None):
        return json.loads(super(RestClientJson, self).get(obj, uid))

    def post(self, obj, data=None):
        return json.loads(super(RestClientJson, self).post(obj, json_dumps(data)))

    def put(self, obj, uid, data=None):
        return json.loads(super(RestClientJson, self).put(obj, uid, json_dumps(data)))

    def delete(self, obj, uid):
        return json.loads(super(RestClientJson, self).delete(obj, uid))
<|endoftext|>


<commit_before>__version__ = '0.1.3'<commit_msg>dev of new version<commit_after>__version__ = '0.1.4-dev'<|endoftext|>