<commit_before>from fabric.api import *
from fabric.contrib.project import rsync_project

try:
    import fabfile_local
except ImportError:
    pass

@task
def deploy(name):
    info = env.deployments[name]
    run('mkdir -p %s' % info['remote_dir'])
    rsync_project(remote_dir=info['remote_dir'],
                  local_dir='data/')
    print "files placed in %s" % info['url']
<commit_msg>added 'deploy_xpi' target to fabfile, based on some other python code I had laying around.<commit_after>from fabric.api import *
from fabric.contrib.project import rsync_project

try:
    import fabfile_local
except ImportError:
    pass

def get_deployment(name):
    return env.deployments[name]

@task
def deploy(name):
    info = get_deployment(name)
    run('mkdir -p %s' % info['remote_dir'])
    rsync_project(remote_dir=info['remote_dir'],
                  local_dir='data/')
    print "files placed in %s" % info['url']

import os
import sys
import datetime
import subprocess
import cgi
import json

CFX = 'cfx'
SCP = 'scp'
HTML_TEMPLATE = """

  <title>{config[fullName]}</title>

{config[fullName]}

{config[description]}

Version {config[version]}

By {config[author]}

Published on {pubdate}

  [ Install Addon |
    View Source ]

"""

@task
def deploy_xpi(name):
    info = get_deployment(name)

    UPDATE_RDF_URL = info['xpi_url'] + "%(update_rdf)s"
    XPI_URL = info['xpi_url'] + "%(xpi)s"
    HTML_URL = info['xpi_url'] + "%(html)s"
    SCP_TARGET = env.hosts[0] + ':' + info['xpi_dir'] + '/'

    pkgdir = os.path.abspath(os.path.dirname(__file__))
    config = json.load(open(os.path.join(pkgdir, 'package.json')))

    update_rdf = '%s.update.rdf' % config['name']
    update_rdf_abspath = os.path.join(pkgdir, update_rdf)
    update_rdf_url = UPDATE_RDF_URL % locals()

    xpi = '%s.xpi' % config['name']
    xpi_abspath = os.path.join(pkgdir, xpi)
    xpi_url = XPI_URL % locals()

    cmdline = [
       CFX,
       'xpi',
       '--pkgdir', pkgdir,
       '--update-url', update_rdf_url,
       '--update-link', xpi_url
       ]

    # Write the template.
    pubdate = str(datetime.datetime.now())
    html = '%s.html' % config['name']
    html_abspath = os.path.join(pkgdir, html)
    for prop in ['fullName', 'description', 'author']:
        config[prop] = cgi.escape(config[prop])
    open(html_abspath, 'w').write(HTML_TEMPLATE.format(**locals()))

    # Build the XPI and update.rdf.
    subprocess.check_call(cmdline)

    # Upload them.
    subprocess.check_call([SCP, xpi_abspath, update_rdf_abspath,
                           html_abspath, SCP_TARGET])

    # Remove them from the local filesystem.
    os.remove(update_rdf_abspath)
    os.remove(xpi_abspath)
    os.remove(html_abspath)

    print "Download the addon at:"
    print HTML_URL % locals()
<|endoftext|>


<commit_before>from time import sleep
import numpy as np
from picamera import PiCamera
from picamera.array import PiRGBAnalysis
from sklearn.cluster import DBSCAN
scan = DBSCAN(eps=2, min_samples=3, metric='euclidean', algorithm='ball_tree', leaf_size=30)

class Analysis(PiRGBAnalysis):
    def __init__(self, camera, size=None):
        self.camera = camera
        self.size = size
        self.x0 = np.array(0)
        self.i = 0
        self.t0 = self.camera.timestamp
        self.xc0 = np.float32(0)
        self.yc0 = np.float32(0)
        self.xc = np.float32(0)
        self.yc = np.float32(0)
        self.xp = np.array(0)
        self.yp = np.array(0)
        self.calibrationMode = True
        self.stableCounter = 0
        self.unstableCounter = 0
        self.bgsum = np.zeros((480, 640), dtype=np.int32)
        self.background = np.array(0, dtype=np.uint8)

    def analyse(self, x):
        x = x[:,:,1]

        if self.calibrationMode:
            d = self.x0-x
            d[self.x0<x] = 0
            if np.any(d>25):
                self.stableCounter = 0
                self.bgsum = 0
            else:
                self.stableCounter += 1
                self.bgsum += x
                if self.stableCounter >= 20:
                    self.background = (self.bgsum/20).round(0).astype(np.uint8)
                    self.stableCounter = 0
                    self.bgsum = 0
                    self.calibrationMode = False
                    print(self.background.mean(), x.mean())
            self.x0 = x

        else:
            d = self.background-x
            d[self.background<x] = 0
            xy = np.argwhere(d>25)
            if xy.shape[0]>2 and xy.shape[0]<500:
                clust = scan.fit_predict(xy)
                ind = clust==0
                if ind.any():
                    self.xc = xy[ind,0].mean()
                    self.yc = xy[ind,1].mean()
                    xc2 = 2 * self.xc - self.xc0
                    yc2 = 2 * self.yc - self.yc0
                    self.xp = np.linspace(self.xc0,xc2,10)
                    self.yp = np.linspace(self.yc0,yc2,10)
                    self.xc0 = self.xc
                    self.yc0 = self.yc
            self.i += 1
            #print("\r" + str(10/td), end="")

camera = PiCamera(resolution=(640, 480), framerate=20)
camera.awb_mode = 'off'
camera.awb_gains = (1.2, 1.2)
#camera.iso = 640
camera.color_effects = (128,128)
camera.exposure_mode = 'sports'
camera.shutter_speed = 12000
camera.video_denoise = False

#fullscreen=False, window=(0,0,640,480)
camera.start_preview()
sleep(1)
tracker = Analysis(camera)
camera.start_recording(tracker, format='rgb')

crosshair = np.zeros((480, 640, 3), dtype=np.uint8)
ol = camera.add_overlay(crosshair, layer=3, alpha=25)
#memoryview tobytes()

hist = [[5,5]] * 30
textPresent = False
#xp0 = 0
#yp0 = 0
try:
    while True:
        if textPresent:
            if not tracker.calibrationMode:
                camera.annotate_text = None
                textPresent = False
        else:
            if tracker.calibrationMode:
                camera.annotate_text = "Calibrating background"
                textPresent = True
        xc = int(tracker.xc.round(0))
        yc = int(tracker.yc.round(0))
        #xp = tracker.xp.round(0).astype(np.int16)
        #yp = tracker.yp.round(0).astype(np.int16)
        if xc!=hist[-1][0] or yc!=hist[-1][1]:
            crosshair[xc, (yc-3):(yc+4), 1] = 0xff
            crosshair[(xc-3):(xc+4), yc, 1] = 0xff
            #try:
            #    crosshair[xp,yp] = 0xff
            #except IndexError:
            #    pass
            #try:
            #    crosshair[xp0,yp0] = 0
            #except IndexError:
            #    pass
            ol.update(crosshair)
            xc0,yc0 = hist.pop(0)
            crosshair[xc0, (yc0-3):(yc0+4), 1] = 0
            crosshair[(xc0-3):(xc0+4), yc0, 1] = 0
            hist.append([xc,yc])
            #xp0 = xp
            #yp0 = yp
        sleep(.05)
except KeyboardInterrupt:
    pass

#camera.start_recording('/home/pi/video2.h264')
#camera.wait_recording(10)
camera.remove_overlay(ol)
camera.stop_recording()
camera.stop_preview()
print("%s frames" % tracker.i)
td = (tracker.camera.timestamp - tracker.t0)/1000000
print("%s fps" % (tracker.i/td))

<commit_msg>unstable background - reenter calibration<commit_after>from time import sleep
import numpy as np
from picamera import PiCamera
from picamera.array import PiRGBAnalysis
from sklearn.cluster import DBSCAN
scan = DBSCAN(eps=2, min_samples=3, metric='euclidean', algorithm='ball_tree', leaf_size=30)

class Analysis(PiRGBAnalysis):
    def __init__(self, camera, size=None):
        self.camera = camera
        self.size = size
        self.x0 = np.array(0)
        self.i = 0
        self.t0 = self.camera.timestamp
        self.xc0 = np.float32(0)
        self.yc0 = np.float32(0)
        self.xc = np.float32(0)
        self.yc = np.float32(0)
        self.xp = np.array(0)
        self.yp = np.array(0)
        self.calibrationMode = True
        self.stableCounter = 0
        self.unstableCounter = 0
        self.bgsum = np.zeros((480, 640), dtype=np.int32)
        self.background = np.array(0, dtype=np.uint8)

    def analyse(self, x):
        x = x[:,:,1]

        if self.calibrationMode:
            d = self.x0-x
            d[self.x0<x] = 0
            if np.any(d>25):
                self.stableCounter = 0
                self.bgsum = 0
            else:
                self.stableCounter += 1
                self.bgsum += x
                if self.stableCounter == 20:
                    self.background = (self.bgsum/self.stableCounter).round(0).astype(np.uint8)
                    self.stableCounter = 0
                    self.bgsum = 0
                    self.calibrationMode = False
                    print(self.background.mean(), x.mean())
            self.x0 = x

        else:
            d = self.background-x
            d[self.background<x] = 0
            xy = np.argwhere(d>25)
            if xy.shape[0]>500:
                self.unstableCounter += 1
                if self.unstableCounter>60:
                    self.calibrationMode = True
                    self.unstableCounter = 0
            else:
                if xy.shape[0]>2:
                    clust = scan.fit_predict(xy)
                    ind = clust==0
                    if ind.any():
                        self.xc = xy[ind,0].mean()
                        self.yc = xy[ind,1].mean()
                        xc2 = 2 * self.xc - self.xc0
                        yc2 = 2 * self.yc - self.yc0
                        self.xp = np.linspace(self.xc0,xc2,10)
                        self.yp = np.linspace(self.yc0,yc2,10)
                        self.xc0 = self.xc
                        self.yc0 = self.yc
        self.i += 1
            #print("\r" + str(10/td), end="")

camera = PiCamera(resolution=(640, 480), framerate=20)
camera.awb_mode = 'off'
camera.awb_gains = (1.2, 1.2)
#camera.iso = 640
camera.color_effects = (128,128)
camera.exposure_mode = 'sports'
camera.shutter_speed = 12000
camera.video_denoise = False

#fullscreen=False, window=(0,0,640,480)
camera.start_preview()
sleep(1)
tracker = Analysis(camera)
camera.start_recording(tracker, format='rgb')

crosshair = np.zeros((480, 640, 3), dtype=np.uint8)
ol = camera.add_overlay(crosshair, layer=3, alpha=25)
#memoryview tobytes()

hist = [[5,5]] * 30
textPresent = False
#xp0 = 0
#yp0 = 0
try:
    while True:
        if textPresent:
            if not tracker.calibrationMode:
                camera.annotate_text = None
                textPresent = False
        else:
            if tracker.calibrationMode:
                camera.annotate_text = "Calibrating background"
                textPresent = True
        xc = int(tracker.xc.round(0))
        yc = int(tracker.yc.round(0))
        #xp = tracker.xp.round(0).astype(np.int16)
        #yp = tracker.yp.round(0).astype(np.int16)
        if xc!=hist[-1][0] or yc!=hist[-1][1]:
            crosshair[xc, (yc-3):(yc+4), 1] = 0xff
            crosshair[(xc-3):(xc+4), yc, 1] = 0xff
            #try:
            #    crosshair[xp,yp] = 0xff
            #except IndexError:
            #    pass
            #try:
            #    crosshair[xp0,yp0] = 0
            #except IndexError:
            #    pass
            ol.update(crosshair)
            xc0,yc0 = hist.pop(0)
            crosshair[xc0, (yc0-3):(yc0+4), 1] = 0
            crosshair[(xc0-3):(xc0+4), yc0, 1] = 0
            hist.append([xc,yc])
            #xp0 = xp
            #yp0 = yp
        sleep(.05)
except KeyboardInterrupt:
    pass

#camera.start_recording('/home/pi/video2.h264')
#camera.wait_recording(10)
camera.remove_overlay(ol)
camera.stop_recording()
camera.stop_preview()
print("%s frames" % tracker.i)
td = (tracker.camera.timestamp - tracker.t0)/1000000
print("%s fps" % (tracker.i/td))

<|endoftext|>


<commit_before>from fabric.api import *

env.hosts = [
    'shaperia@lynx.uberspace.de'
]

env.target_directory = './happyman'

def init():
    run('git clone -q https://github.com/skyshaper/happyman.git ' + env.target_directory)
    with cd(env.target_directory):
        run('virtualenv ./python/virtualenv')

def deploy():
    local('git push')
    with cd(env.target_directory):
        run('git remote update && git reset --hard origin/master')
        run('./vendor/bin/carton install --cached --deployment')
        run('./python/virtualenv/bin/pip install --no-index --find-links=python/vendor/cache/ -r python/requirements_lock.txt')
    execute(restart)

def restart():
    run('svc -t ~/service/happyman')
<commit_msg>Do not install develop and test modules in production<commit_after>from fabric.api import *

env.hosts = [
    'shaperia@lynx.uberspace.de'
]

env.target_directory = './happyman'

def init():
    run('git clone -q https://github.com/skyshaper/happyman.git ' + env.target_directory)
    with cd(env.target_directory):
        run('virtualenv ./python/virtualenv')

def deploy():
    local('git push')
    with cd(env.target_directory):
        run('git remote update && git reset --hard origin/master')
        run('./vendor/bin/carton install --cached --deployment --without develop,test')
        run('./python/virtualenv/bin/pip install --no-index --find-links=python/vendor/cache/ -r python/requirements_lock.txt')
    execute(restart)

def restart():
    run('svc -t ~/service/happyman')
<|endoftext|>


<commit_before>from clld.deploy import tasks
tasks.init('glottolog3')
<commit_msg>adapted to new package structure<commit_after>from clldfabric import tasks
tasks.init('glottolog3')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from PyQt4.QtGui import *
from PyQt4.QtCore import *

from ui.gitview import *
from common import *

import subprocess

class GitView(QWidget):
    reqCommit = pyqtSignal()
    reqFind = pyqtSignal()

    def __init__(self, parent=None):
        super(GitView, self).__init__(parent)

        self.ui = Ui_GitView()
        self.ui.setupUi(self)
        self.repo = None
        self.pattern = None

        height = self.ui.splitter.sizeHint().height()
        sizes = [height * 1 / 4, height * 3 / 4]
        self.ui.splitter.setSizes(sizes)

        self.ui.cbBranch.currentIndexChanged.connect(self.__onBranchChanged)
        self.ui.logView.clicked.connect(self.__onCommitChanged)
        self.ui.btnFind.clicked.connect(self.__onBtnFindClicked)

        self.reqCommit.connect(self.__onReqCommit)
        self.reqFind.connect(self.__onBtnFindClicked)

        self.ui.leSha1.installEventFilter(self)
        self.ui.leFindWhat.installEventFilter(self)

    def __updateBranches(self):
        self.ui.cbBranch.clear()
        self.ui.logView.clear()
        self.ui.diffView.clear()
        self.ui.leSha1.clear()

        if not self.repo:
            return

        data = subprocess.check_output(["git", "branch", "-a"])
        data = data.decode("utf-8")
        branches = data.split('\n')

        curBranchIdx = -1
        self.ui.cbBranch.blockSignals(True)

        for branch in branches:
            branch = branch.strip()
            if branch.startswith("remotes/origin/"):
                if not branch.startswith("remotes/origin/HEAD"):
                    self.ui.cbBranch.addItem(branch)
            elif branch:
                if branch.startswith("*"):
                    self.ui.cbBranch.addItem(branch.replace("*", "").strip())
                    curBranchIdx = self.ui.cbBranch.count() - 1
                else:
                    self.ui.cbBranch.addItem(branch.strip())

        if curBranchIdx != -1:
            self.ui.cbBranch.setCurrentIndex(curBranchIdx)

        self.ui.cbBranch.blockSignals(False)
        branchIdx = self.ui.cbBranch.currentIndex()
        self.__onBranchChanged(branchIdx)

    def __onBranchChanged(self, index):
        if index == -1 or self.ui.cbBranch.count() == 0:
            return

        self.ui.logView.clear()
        self.ui.diffView.clear()

        curBranch = self.ui.cbBranch.currentText()
        args = ["git", "log", "-z",
                "--pretty=format:{0}".format(log_fmt),
                curBranch]
        if self.pattern:
            args.append(self.pattern)

        process = subprocess.Popen(args,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        data = process.communicate()[0]

        if process.returncode is 0 and data:
            commits = data.decode("utf-8").split("\0")
            self.ui.logView.setLogs(commits)

    def __onCommitChanged(self, index):
        if index.isValid():
            commit = index.data(Qt.UserRole)
            self.ui.leSha1.setText(commit.sha1)
            self.ui.diffView.showCommit(commit)
        else:
            self.ui.leSha1.clear()
            self.ui.diffView.clear()

    def __onBtnFindClicked(self, checked):
        pass

    def __onReqCommit(self):
        sha1 = self.ui.leSha1.text().strip()
        if sha1:
            ok = self.ui.logView.switchToCommit(sha1)
            if not ok:
                self.window().showMessage(
                    self.tr("Revision '{0}' is not known".format(sha1)))

    def __filterLog(self, pattern):
        if pattern != self.pattern:
            self.pattern = pattern
            index = self.ui.cbBranch.currentIndex()
            self.__onBranchChanged(index)

    def setBranchDesc(self, desc):
        self.ui.lbBranch.setText(desc)

    def setRepo(self, repo):
        if self.repo != repo:
            self.repo = repo
            self.__updateBranches()

    def filterPath(self, path):
        if not path:
            newPattern = None
        else:
            newPattern = path

        self.ui.diffView.setFilterPath(path)
        self.__filterLog(newPattern)

    def filterCommit(self, pattern):
        if not pattern:
            newPattern = None
        else:
            newPattern = "--grep={0}".format(pattern)

        self.ui.diffView.setFilterPath(None)
        self.__filterLog(newPattern)

    def eventFilter(self, obj, event):
        if event.type() == QEvent.KeyRelease and event.key() == Qt.Key_Return:
            if obj == self.ui.leSha1:
                self.reqCommit.emit()
            elif obj == self.ui.leFindWhat:
                self.reqFind.emit()

        return super(GitView, self).eventFilter(obj, event)
<commit_msg>Fixed diffview not update when logview selection changed by keyboard.<commit_after># -*- coding: utf-8 -*-

from PyQt4.QtGui import *
from PyQt4.QtCore import *

from ui.gitview import *
from common import *

import subprocess

class GitView(QWidget):
    reqCommit = pyqtSignal()
    reqFind = pyqtSignal()

    def __init__(self, parent=None):
        super(GitView, self).__init__(parent)

        self.ui = Ui_GitView()
        self.ui.setupUi(self)
        self.repo = None
        self.pattern = None

        height = self.ui.splitter.sizeHint().height()
        sizes = [height * 1 / 4, height * 3 / 4]
        self.ui.splitter.setSizes(sizes)

        self.ui.cbBranch.currentIndexChanged.connect(self.__onBranchChanged)
        self.ui.logView.selectionModel().currentChanged.connect(self.__onCommitChanged)
        self.ui.btnFind.clicked.connect(self.__onBtnFindClicked)

        self.reqCommit.connect(self.__onReqCommit)
        self.reqFind.connect(self.__onBtnFindClicked)

        self.ui.leSha1.installEventFilter(self)
        self.ui.leFindWhat.installEventFilter(self)

    def __updateBranches(self):
        self.ui.cbBranch.clear()
        self.ui.logView.clear()
        self.ui.diffView.clear()
        self.ui.leSha1.clear()

        if not self.repo:
            return

        data = subprocess.check_output(["git", "branch", "-a"])
        data = data.decode("utf-8")
        branches = data.split('\n')

        curBranchIdx = -1
        self.ui.cbBranch.blockSignals(True)

        for branch in branches:
            branch = branch.strip()
            if branch.startswith("remotes/origin/"):
                if not branch.startswith("remotes/origin/HEAD"):
                    self.ui.cbBranch.addItem(branch)
            elif branch:
                if branch.startswith("*"):
                    self.ui.cbBranch.addItem(branch.replace("*", "").strip())
                    curBranchIdx = self.ui.cbBranch.count() - 1
                else:
                    self.ui.cbBranch.addItem(branch.strip())

        if curBranchIdx != -1:
            self.ui.cbBranch.setCurrentIndex(curBranchIdx)

        self.ui.cbBranch.blockSignals(False)
        branchIdx = self.ui.cbBranch.currentIndex()
        self.__onBranchChanged(branchIdx)

    def __onBranchChanged(self, index):
        if index == -1 or self.ui.cbBranch.count() == 0:
            return

        self.ui.logView.clear()
        self.ui.diffView.clear()

        curBranch = self.ui.cbBranch.currentText()
        args = ["git", "log", "-z",
                "--pretty=format:{0}".format(log_fmt),
                curBranch]
        if self.pattern:
            args.append(self.pattern)

        process = subprocess.Popen(args,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        data = process.communicate()[0]

        if process.returncode is 0 and data:
            commits = data.decode("utf-8").split("\0")
            self.ui.logView.setLogs(commits)

    def __onCommitChanged(self, current, previous):
        if current.isValid():
            commit = current.data(Qt.UserRole)
            self.ui.leSha1.setText(commit.sha1)
            self.ui.diffView.showCommit(commit)
        else:
            self.ui.leSha1.clear()
            self.ui.diffView.clear()

    def __onBtnFindClicked(self, checked):
        pass

    def __onReqCommit(self):
        sha1 = self.ui.leSha1.text().strip()
        if sha1:
            ok = self.ui.logView.switchToCommit(sha1)
            if not ok:
                self.window().showMessage(
                    self.tr("Revision '{0}' is not known".format(sha1)))

    def __filterLog(self, pattern):
        if pattern != self.pattern:
            self.pattern = pattern
            index = self.ui.cbBranch.currentIndex()
            self.__onBranchChanged(index)

    def setBranchDesc(self, desc):
        self.ui.lbBranch.setText(desc)

    def setRepo(self, repo):
        if self.repo != repo:
            self.repo = repo
            self.__updateBranches()

    def filterPath(self, path):
        if not path:
            newPattern = None
        else:
            newPattern = path

        self.ui.diffView.setFilterPath(path)
        self.__filterLog(newPattern)

    def filterCommit(self, pattern):
        if not pattern:
            newPattern = None
        else:
            newPattern = "--grep={0}".format(pattern)

        self.ui.diffView.setFilterPath(None)
        self.__filterLog(newPattern)

    def eventFilter(self, obj, event):
        if event.type() == QEvent.KeyRelease and event.key() == Qt.Key_Return:
            if obj == self.ui.leSha1:
                self.reqCommit.emit()
            elif obj == self.ui.leFindWhat:
                self.reqFind.emit()

        return super(GitView, self).eventFilter(obj, event)
<|endoftext|>


<commit_before>"""Fabfile for the ``webfaction-django-boilerplate``.

Make sure to setup your ``fabric_settings.py`` first. As a start, just copy
``fabric_settings.py.sample``.

"""
from __future__ import with_statement

from fabric.api import (
    cd,
    env,
    lcd,
    local,
    run,
    settings,
)
from fabric.contrib.files import append, exists,  sed

import fabric_settings as fab_settings

env.hosts = fab_settings.ENV_HOSTS
env.user = fab_settings.ENV_USER

BASHRC_SETTING1 = 'export VIRTUALENVWRAPPER_PYTHON=/usr/local/bin/python2.7'
BASHRC_SETTING2 = 'export WORKON_HOME=$HOME/Envs'
BASHRC_SETTING3 = 'source /home/{0}/bin/virtualenvwrapper.sh'.format(env.user)
BASHRC_SETTING4 = 'export PIP_VIRTUALENV_BASE=$WORKON_HOME'
BASHRC_SETTING5 = 'export PIP_RESPECT_VIRTUALENV=true'

# ****************************************************************************
# HIGH LEVEL TASKS
# ****************************************************************************
def install_everything():
    install_local_repo()
    install_server()
    local_link_repo_with_remote_repo()
    first_deployment()

def first_deployment():
    run_clone_repo()
    run_install_scripts()
    run_prepare_wsgi()
    run_install_requirements()
    run_prepare_local_settings()

def install_local_repo():
    local_create_new_repo()
    local_init_django_project()
    local_initial_commit()

def install_server():
    run_install_virtualenv()
    run_install_mercurial()
    run_add_bashrc_settings()
    run_create_virtualenv()
    run_create_git_repo()

# ****************************************************************************
# LOCAL TASKS
# ****************************************************************************
def local_link_repo_with_remote_repo():
    with lcd(fab_settings.PROJECT_ROOT):
        local('git config http.sslVerify false')
        local('git config http.postBuffer 524288000')
        local('git remote add origin'
                ' https://{0}@git.{0}.webfactional.com/{1}'.format(
                    fab_settings.ENV_USER, fab_settings.GIT_REPO_NAME))
        local('git push -u origin master')

def local_create_new_repo():
    with lcd(fab_settings.PROJECT_ROOT):
        local('rm -rf .git')
        local('rm .gitmodules')
        local('rm -rf website/webapps/django/project/submodules/Skeleton')
        local('git init')
        local('git submodule add git://github.com/dhgamache/Skeleton.git'
                ' website/webapps/django/project/submodules/Skeleton')

def local_init_django_project():
    with lcd(fab_settings.DJANGO_PROJECT_ROOT):
        local('cp settings/local/local_settings.py.sample'
                ' settings/local/local_settings.py')
        local('cp settings/local/gorun_settings.py.sample gorun_settings.py')
        local('python manage.py syncdb --all --noinput')
        local('python manage.py migrate --fake')
        local('python manage.py loaddata bootstrap_auth.json')

def local_initial_commit():
    with lcd(fab_settings.PROJECT_ROOT):
        local('git add .')
        local('git commit -am "Initial commit."')

# ****************************************************************************
# REMOTE TASKS
# ****************************************************************************
def run_add_bashrc_settings():
    with cd('$HOME'):
        append('.bashrc', BASHRC_SETTING1, partial=True)
        append('.bashrc', BASHRC_SETTING2, partial=True)
        append('.bashrc', BASHRC_SETTING3, partial=True)
        append('.bashrc', BASHRC_SETTING4, partial=True)
        append('.bashrc', BASHRC_SETTING5, partial=True)

def run_clone_repo():
    run('mkdir -p $HOME/src')
    cloned_repo_path = '$HOME/src/{0}'.format(fab_settings.PROJECT_NAME)
    if exists(cloned_repo_path):
        run('rm -rf {0}'.format(cloned_repo_path))
    with cd('$HOME/src'):
        run('git clone $HOME/webapps/git/repos/{0} {1}'.format(
            fab_settings.GIT_REPO_NAME, fab_settings.PROJECT_NAME))

def run_create_git_repo():
    run('rm -rf $HOME/webapps/git/repos/{0}'.format(
        fab_settings.GIT_REPO_NAME))
    with cd('$HOME/webapps/git'):
        run('git init --bare ./repos/{0}'.format(fab_settings.GIT_REPO_NAME))
    with cd('$HOME/webapps/git/repos/{0}'.format(fab_settings.GIT_REPO_NAME)):
        run('git config http.receivepack true')

def run_create_ssh_dir():
    with cd('$HOME'):
        with settings(warn_only=True):
            run('mkdir .ssh')
            run('touch .ssh/authorized_keys')
            run('chmod 600 .ssh/authorized_keys')
            run('chmod 700 .ssh')

def run_create_virtualenv():
    with cd('$HOME'):
        run('rm -rf $HOME/Envs/{0}'.format(fab_settings.VENV_NAME))
        run('mkvirtualenv -p python2.7 --system-site-packages {0}'.format(
            fab_settings.VENV_NAME))

def run_install_mercurial():
    with cd('$HOME'):
        run('easy_install-2.7 mercurial')

def run_install_requirements():
    run('workon {0} && pip install -r $HOME/src/{1}/website/webapps/django/'
        'project/requirements.txt --upgrade'.format(
            fab_settings.VENV_NAME, fab_settings.PROJECT_NAME))

def run_install_scripts():
    with cd('$HOME/src/{0}/scripts'.format(fab_settings.PROJECT_NAME)):
        run('git pull origin master')
        run('cp deploy-website.sh $HOME/bin/deploy-website-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp mysql-backup.sh $HOME/bin/mysql-backup-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp restart-apache.sh $HOME/bin/restart-apache-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp show-memory.sh $HOME/bin/show-memory.sh')
        run('cp script_settings.sh $HOME/bin/script_settings.sh')

    with cd('$HOME/bin'):
        sed('script_settings.sh', 'INSERT_USERNAME', fab_settings.ENV_USER)
        sed('script_settings.sh', 'INSERT_DB_USER', fab_settings.MYSQL_DB_USER)
        sed('script_settings.sh', 'INSERT_DB_NAME', fab_settings.MYSQL_DB_NAME)
        sed('script_settings.sh', 'INSERT_DB_PASSWORD',
            fab_settings.MYSQL_DB_PASSWORD)
        sed('script_settings.sh', 'INSERT_PROJECT_NAME',
            fab_settings.PROJECT_NAME)
        sed('script_settings.sh', 'INSERT_DJANGO_APP_NAME',
            fab_settings.DJANGO_APP_NAME)
        sed('script_settings.sh', 'INSERT_VENV_NAME',
            fab_settings.VENV_NAME)

def run_install_virtualenv():
    with cd('$HOME'):
        run('mkdir -p $HOME/lib/python2.7')
        run('easy_install-2.7 virtualenv')
        run('easy_install-2.7 pip')
        run('pip install virtualenvwrapper')
        run('mkdir -p $HOME/Envs')

def run_prepare_local_settings():
    with cd('$HOME/webapps/{0}/project/settings/local'.format(
        fab_settings.DJANGO_APP_NAME)):
        run('cp local_settings.py.sample local_settings.py')
        sed('local_settings.py', 'backends.sqlite3', 'backends.mysql')
        sed('local_settings.py', 'db.sqlite', fab_settings.MYSQL_DB_NAME)
        sed('local_settings.py', '"USER": ""', '"USER": "{0}"'.format(
            fab_settings.MYSQL_DB_USER))
        sed('local_settings.py', '"PASSWORD": ""', '"PASSWORD": "{0}"'.format(
            fab_settings.MYSQL_DB_PASSWORD))
        sed('local_settings.py', 'yourproject', '{0}'.format(
            fab_settings.PROJECT_NAME))
        sed('local_settings.py', 'EMAIL_BACKEND', '#EMAIL_BACKEND')
        sed('local_settings.py', '##EMAIL_BACKEND', 'EMAIL_BACKEND')
        sed('local_settings.py', '#EMAIL_HOST', 'EMAIL_HOST')
        sed('local_settings.py', '#EMAIL_HOST_USER', 'EMAIL_HOST_USER')
        sed('local_settings.py', 'EMAIL_HOST_USER = ""',
            'EMAIL_HOST_USER = "{0}"'.format(fab_settings.EMAIL_INBOX))
        sed('local_settings.py', '#EMAIL_HOST_PASSWORD', 'EMAIL_HOST_PASSWORD')
        sed('local_settings.py', 'EMAIL_HOST_PASSWORD = ""',
            'EMAIL_HOST_PASSWORD = "{0}"'.format(fab_settings.EMAIL_PASSWORD))
        sed('local_settings.py', '#EMAIL_USE_TLS', 'EMAIL_USE_TLS')
        sed('local_settings.py', '#EMAIL_PORT', 'EMAIL_PORT')
        sed('local_settings.py', 'MEDIA_APP_NAME', fab_settings.MEDIA_APP_NAME)
        sed('local_settings.py', 'STATIC_APP_NAME',
            fab_settings.STATIC_APP_NAME)
        sed('local_settings.py', 'yourname', fab_settings.ADMIN_NAME)
        sed('local_settings.py', 'info@example.com', fab_settings.ADMIN_EMAIL)

def run_prepare_wsgi():
    with cd('$HOME/webapps/{0}/lib/python2.7/'.format(
        fab_settings.DJANGO_APP_NAME)):
        run('rm -rf django')
        run('rm -rf Django*')
    with cd('$HOME/webapps/{0}'.format(fab_settings.DJANGO_APP_NAME)):
        run('rm -rf myproject')
        run('cp $HOME/src/{0}/scripts/myproject.wsgi .'.format(
            fab_settings.PROJECT_NAME))
        sed('myproject.wsgi', 'ENV_USER', fab_settings.ENV_USER)
        sed('myproject.wsgi', 'VENV_NAME', fab_settings.VENV_NAME)
        sed('myproject.wsgi', 'DJANGO_APP_NAME', fab_settings.DJANGO_APP_NAME)
<commit_msg>Initiating submodule after first clone on server.<commit_after>"""Fabfile for the ``webfaction-django-boilerplate``.

Make sure to setup your ``fabric_settings.py`` first. As a start, just copy
``fabric_settings.py.sample``.

"""
from __future__ import with_statement

from fabric.api import (
    cd,
    env,
    lcd,
    local,
    run,
    settings,
)
from fabric.contrib.files import append, exists,  sed

import fabric_settings as fab_settings

env.hosts = fab_settings.ENV_HOSTS
env.user = fab_settings.ENV_USER

BASHRC_SETTING1 = 'export VIRTUALENVWRAPPER_PYTHON=/usr/local/bin/python2.7'
BASHRC_SETTING2 = 'export WORKON_HOME=$HOME/Envs'
BASHRC_SETTING3 = 'source /home/{0}/bin/virtualenvwrapper.sh'.format(env.user)
BASHRC_SETTING4 = 'export PIP_VIRTUALENV_BASE=$WORKON_HOME'
BASHRC_SETTING5 = 'export PIP_RESPECT_VIRTUALENV=true'

# ****************************************************************************
# HIGH LEVEL TASKS
# ****************************************************************************
def install_everything():
    install_local_repo()
    install_server()
    local_link_repo_with_remote_repo()
    first_deployment()

def first_deployment():
    run_clone_repo()
    run_install_scripts()
    run_prepare_wsgi()
    run_install_requirements()
    run_prepare_local_settings()

def install_local_repo():
    local_create_new_repo()
    local_init_django_project()
    local_initial_commit()

def install_server():
    run_install_virtualenv()
    run_install_mercurial()
    run_add_bashrc_settings()
    run_create_virtualenv()
    run_create_git_repo()

# ****************************************************************************
# LOCAL TASKS
# ****************************************************************************
def local_link_repo_with_remote_repo():
    with lcd(fab_settings.PROJECT_ROOT):
        local('git config http.sslVerify false')
        local('git config http.postBuffer 524288000')
        local('git remote add origin'
                ' https://{0}@git.{0}.webfactional.com/{1}'.format(
                    fab_settings.ENV_USER, fab_settings.GIT_REPO_NAME))
        local('git push -u origin master')

def local_create_new_repo():
    with lcd(fab_settings.PROJECT_ROOT):
        local('rm -rf .git')
        local('rm .gitmodules')
        local('rm -rf website/webapps/django/project/submodules/Skeleton')
        local('git init')
        local('git submodule add git://github.com/dhgamache/Skeleton.git'
                ' website/webapps/django/project/submodules/Skeleton')

def local_init_django_project():
    with lcd(fab_settings.DJANGO_PROJECT_ROOT):
        local('cp settings/local/local_settings.py.sample'
                ' settings/local/local_settings.py')
        local('cp settings/local/gorun_settings.py.sample gorun_settings.py')
        local('python manage.py syncdb --all --noinput')
        local('python manage.py migrate --fake')
        local('python manage.py loaddata bootstrap_auth.json')

def local_initial_commit():
    with lcd(fab_settings.PROJECT_ROOT):
        local('git add .')
        local('git commit -am "Initial commit."')

# ****************************************************************************
# REMOTE TASKS
# ****************************************************************************
def run_add_bashrc_settings():
    with cd('$HOME'):
        append('.bashrc', BASHRC_SETTING1, partial=True)
        append('.bashrc', BASHRC_SETTING2, partial=True)
        append('.bashrc', BASHRC_SETTING3, partial=True)
        append('.bashrc', BASHRC_SETTING4, partial=True)
        append('.bashrc', BASHRC_SETTING5, partial=True)

def run_clone_repo():
    run('mkdir -p $HOME/src')
    cloned_repo_path = '$HOME/src/{0}'.format(fab_settings.PROJECT_NAME)
    if exists(cloned_repo_path):
        run('rm -rf {0}'.format(cloned_repo_path))
    with cd('$HOME/src'):
        run('git clone $HOME/webapps/git/repos/{0} {1}'.format(
            fab_settings.GIT_REPO_NAME, fab_settings.PROJECT_NAME))
    with cd('$HOME/src/{0}'.format(fab_settings.PROJECT_NAME)):
        run('git submodule init')
        run('git submodule update')

def run_create_git_repo():
    run('rm -rf $HOME/webapps/git/repos/{0}'.format(
        fab_settings.GIT_REPO_NAME))
    with cd('$HOME/webapps/git'):
        run('git init --bare ./repos/{0}'.format(fab_settings.GIT_REPO_NAME))
    with cd('$HOME/webapps/git/repos/{0}'.format(fab_settings.GIT_REPO_NAME)):
        run('git config http.receivepack true')

def run_create_ssh_dir():
    with cd('$HOME'):
        with settings(warn_only=True):
            run('mkdir .ssh')
            run('touch .ssh/authorized_keys')
            run('chmod 600 .ssh/authorized_keys')
            run('chmod 700 .ssh')

def run_create_virtualenv():
    with cd('$HOME'):
        run('rm -rf $HOME/Envs/{0}'.format(fab_settings.VENV_NAME))
        run('mkvirtualenv -p python2.7 --system-site-packages {0}'.format(
            fab_settings.VENV_NAME))

def run_install_mercurial():
    with cd('$HOME'):
        run('easy_install-2.7 mercurial')

def run_install_requirements():
    run('workon {0} && pip install -r $HOME/src/{1}/website/webapps/django/'
        'project/requirements.txt --upgrade'.format(
            fab_settings.VENV_NAME, fab_settings.PROJECT_NAME))

def run_install_scripts():
    with cd('$HOME/src/{0}/scripts'.format(fab_settings.PROJECT_NAME)):
        run('git pull origin master')
        run('cp deploy-website.sh $HOME/bin/deploy-website-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp mysql-backup.sh $HOME/bin/mysql-backup-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp restart-apache.sh $HOME/bin/restart-apache-{0}.sh'.format(
            fab_settings.PROJECT_NAME))
        run('cp show-memory.sh $HOME/bin/show-memory.sh')
        run('cp script_settings.sh $HOME/bin/script_settings.sh')

    with cd('$HOME/bin'):
        sed('script_settings.sh', 'INSERT_USERNAME', fab_settings.ENV_USER)
        sed('script_settings.sh', 'INSERT_DB_USER', fab_settings.MYSQL_DB_USER)
        sed('script_settings.sh', 'INSERT_DB_NAME', fab_settings.MYSQL_DB_NAME)
        sed('script_settings.sh', 'INSERT_DB_PASSWORD',
            fab_settings.MYSQL_DB_PASSWORD)
        sed('script_settings.sh', 'INSERT_PROJECT_NAME',
            fab_settings.PROJECT_NAME)
        sed('script_settings.sh', 'INSERT_DJANGO_APP_NAME',
            fab_settings.DJANGO_APP_NAME)
        sed('script_settings.sh', 'INSERT_VENV_NAME',
            fab_settings.VENV_NAME)

def run_install_virtualenv():
    with cd('$HOME'):
        run('mkdir -p $HOME/lib/python2.7')
        run('easy_install-2.7 virtualenv')
        run('easy_install-2.7 pip')
        run('pip install virtualenvwrapper')
        run('mkdir -p $HOME/Envs')

def run_prepare_local_settings():
    with cd('$HOME/webapps/{0}/project/settings/local'.format(
        fab_settings.DJANGO_APP_NAME)):
        run('cp local_settings.py.sample local_settings.py')
        sed('local_settings.py', 'backends.sqlite3', 'backends.mysql')
        sed('local_settings.py', 'db.sqlite', fab_settings.MYSQL_DB_NAME)
        sed('local_settings.py', '"USER": ""', '"USER": "{0}"'.format(
            fab_settings.MYSQL_DB_USER))
        sed('local_settings.py', '"PASSWORD": ""', '"PASSWORD": "{0}"'.format(
            fab_settings.MYSQL_DB_PASSWORD))
        sed('local_settings.py', 'yourproject', '{0}'.format(
            fab_settings.PROJECT_NAME))
        sed('local_settings.py', 'EMAIL_BACKEND', '#EMAIL_BACKEND')
        sed('local_settings.py', '##EMAIL_BACKEND', 'EMAIL_BACKEND')
        sed('local_settings.py', '#EMAIL_HOST', 'EMAIL_HOST')
        sed('local_settings.py', '#EMAIL_HOST_USER', 'EMAIL_HOST_USER')
        sed('local_settings.py', 'EMAIL_HOST_USER = ""',
            'EMAIL_HOST_USER = "{0}"'.format(fab_settings.EMAIL_INBOX))
        sed('local_settings.py', '#EMAIL_HOST_PASSWORD', 'EMAIL_HOST_PASSWORD')
        sed('local_settings.py', 'EMAIL_HOST_PASSWORD = ""',
            'EMAIL_HOST_PASSWORD = "{0}"'.format(fab_settings.EMAIL_PASSWORD))
        sed('local_settings.py', '#EMAIL_USE_TLS', 'EMAIL_USE_TLS')
        sed('local_settings.py', '#EMAIL_PORT', 'EMAIL_PORT')
        sed('local_settings.py', 'MEDIA_APP_NAME', fab_settings.MEDIA_APP_NAME)
        sed('local_settings.py', 'STATIC_APP_NAME',
            fab_settings.STATIC_APP_NAME)
        sed('local_settings.py', 'yourname', fab_settings.ADMIN_NAME)
        sed('local_settings.py', 'info@example.com', fab_settings.ADMIN_EMAIL)

def run_prepare_wsgi():
    with cd('$HOME/webapps/{0}/lib/python2.7/'.format(
        fab_settings.DJANGO_APP_NAME)):
        run('rm -rf django')
        run('rm -rf Django*')
    with cd('$HOME/webapps/{0}'.format(fab_settings.DJANGO_APP_NAME)):
        run('rm -rf myproject')
        run('cp $HOME/src/{0}/scripts/myproject.wsgi .'.format(
            fab_settings.PROJECT_NAME))
        sed('myproject.wsgi', 'ENV_USER', fab_settings.ENV_USER)
        sed('myproject.wsgi', 'VENV_NAME', fab_settings.VENV_NAME)
        sed('myproject.wsgi', 'DJANGO_APP_NAME', fab_settings.DJANGO_APP_NAME)
<|endoftext|>


<commit_before>"""

    glc.gif
    =============

    (c) 2016 LeoV
    https://github.com/leovoel/

"""

from subprocess import Popen, DEVNULL, PIPE
from .config import IMAGEMAGICK_BINARY, FFMPEG_BINARY
from .animation import Animation

import os
import shlex
import imageio

class Gif(Animation):

    """Animation rendered to a gif file.

    This is a subclass of :class:`Animation`.

    Parameters
    ----------
    color_count : int
        Power of two value that indicates the size of the palette for the gif.
        Defaults to 256.
    converter : str
        The converter to use. Right now, there are three converters:
        - ``'imagemagick'``
        - ``'imagemagick_tempfiles'``
        - ``'imageio'``

        The ``imagemagick[_tempfiles]`` converters are the
        only ones that support transparent gifs.

        Defaults to ``'imageio'``.
    converter_opts : dict
        Dictionary with options for the converters.
    """

    def __init__(self, filename, *args, **kwargs):
        super().__init__(filename, *args, **kwargs)
        self.color_count = kwargs.get("color_count", 256)
        self.converter = kwargs.get("converter", "imageio")
        self.converter_opts = kwargs.get("converter_opts", dict())

    def set_converter_opts(self, **kwargs):
        """Sets the options for the current converter."""
        self.converter_opts.update(kwargs)
        return self

    def save_with_imagemagick_tempfiles(self, frames, filename):
        """Writes this animation to a GIF file using ImageMagick, using temporary files.

        This exporter supports transparent backgrounds.
        This saves every frame to a temporary file.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """
        # this is stupid

        _filename, _file_extension = os.path.splitext(filename)

        temp_filenames = []

        index = 0

        for frame in frames:
            temp_name = "{}_TEMP_{:04d}.png".format(_filename, index)
            temp_filenames.append(temp_name)
            imageio.imsave(temp_name, frame)
            index += 1

        delay = int(100 / self.fps)
        fuzz = self.convert_opts.get("fuzz", 1)
        layer_opt = self.converter_opts.get("layer_opt", "OptimizeTransparency")

        cmd = [
            IMAGEMAGICK_BINARY,
            "-delay", str(delay),
            "-dispose", "{:d}".format(2 if self.converter_opts.get("dispose", False) or self.transparent else 1),
            "-loop", "{:d}".format(self.converter_opts.get("loop", 0)),
            "{}_TEMP_*.png".format(_filename),
            "-coalesce",
            "-layers", layer_opt,
            "-colors", str(self.color_count),
            "-fuzz", "{:02d}%".format(fuzz),
            filename
        ]

        proc = Popen(cmd)
        proc.wait()

        for f in temp_filenames:
            os.remove(f)

    def save_with_imagemagick(self, frames, filename):
        """Writes this animation to a GIF file using ImageMagick and FFmpeg.

        This exporter supports transparent backgrounds.
        This uses piping to avoid temporary files.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """

        # NOTE: main idea here is to grab frames using ffmpeg,
        # and pipe those to imagemagick's convert.
        # the reason we don't just use ffmpeg is because
        # it produces incredibly grainy gifs by default,
        # and most people on the web say that the best way is
        # to use imagemagick *shrugs*
        # this is based on moviepy's gif writers
        # (https://github.com/Zulko/moviepy/blob/master/moviepy/video/io/gif_writers.py)
        # thanks!

        delay = 100.0 / self.fps

        # NOTE: for the last -vcodec, ppm or bmp works, png doesn't???
        # ppm is fine for non transparent stuff,
        # otherwise bmp is the one that works properly.
        # see also: http://www.ffmpeg.org/general.html#toc-Image-Formats

        ffmpeg_command = [
            FFMPEG_BINARY,
            "-y",
            "-loglevel", "error",
            "-f", "rawvideo",
            "-vcodec", "rawvideo",
            "-r", "{:.02f}".format(self.fps),
            "-s", "{:d}x{:d}".format(self.w, self.h),
            "-pix_fmt", "rgba",
            "-i", "-",
            "-f", "image2pipe",
            "-vcodec", ("bmp" if self.transparent else "ppm"),
            "-"
        ]

        popen_kwargs = {"stdout": DEVNULL, "stderr": DEVNULL, "stdin": DEVNULL}

        # NOTE: CREATE_NO_WINDOW
        # see https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863%28v=vs.85%29.aspx
        if os.name == "nt":
            popen_kwargs["creationflags"] = 0x08000000

        popen_kwargs["stdin"] = PIPE
        popen_kwargs["stdout"] = PIPE

        ffmpeg_process = Popen(ffmpeg_command, **popen_kwargs)

        # RE: dispose
        # see http://www.imagemagick.org/script/command-line-options.php#dispose
        # we use 2 if the animation is meant to be transparent
        # otherwise you're on your own, and should set the "dispose" option
        # in the converter_opts arg in the constructor, or set_converter_opts
        im_command = [
            IMAGEMAGICK_BINARY,
            "-delay", "{:.02f}".format(delay),
            "-dispose", "{:d}".format(2 if self.converter_opts.get("dispose", False) or self.transparent else 1),
            "-loop", "{:d}".format(self.converter_opts.get("loop", 0)),
        ]

        im_command.extend(shlex.split(self.converter_opts.get("before_args", "")))
        im_command.extend(["-", "-coalesce", filename])

        popen_kwargs["stdin"] = ffmpeg_process.stdout
        popen_kwargs["stdout"] = DEVNULL
        im_process = Popen(im_command, **popen_kwargs)

        for frame in frames:
            ffmpeg_process.stdin.write(frame.tostring())

        ffmpeg_process.stdin.close()
        ffmpeg_process.wait()

        im_process.wait()

    def save_with_imageio(self, frames, filename):
        """Writes this animation to a GIF file using imageio.

        This exporter does not support transparent backgrounds.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """
        # TODO: make this more customizable

        quant = self.converter_opts.get("quantizer", "wu")

        writer = imageio.get_writer(
            filename,
            duration=1 / self.fps,
            quantizer=quant,
            palettesize=self.color_count
        )

        for frame in frames:
            writer.append_data(frame)

        writer.close()

    def render_and_save(self, filename=None):
        """Renders the animation and writes it to a GIF file.

        This uses the currently set exporter based on the attribute ``converter``.
        If ``converter`` is somehow ``None``, or some unknown value, the default
        is to export the animation using imageio.

        Parameters
        ----------
        filename : str
            The filename to use when saving the file.
        """
        if filename is None:
            filename = self.filename

        frames = self.render_all()
        func = getattr(self, "save_with_{}".format(self.converter), None)

        if func is None:
            self.save_with_imageio(frames, filename)
            return
        func(frames, filename)
<commit_msg>fix AttributeError in save_with_imagemagick_tempfiles<commit_after>"""

    glc.gif
    =============

    (c) 2016 LeoV
    https://github.com/leovoel/

"""

from subprocess import Popen, DEVNULL, PIPE
from .config import IMAGEMAGICK_BINARY, FFMPEG_BINARY
from .animation import Animation

import os
import shlex
import imageio

class Gif(Animation):

    """Animation rendered to a gif file.

    This is a subclass of :class:`Animation`.

    Parameters
    ----------
    color_count : int
        Power of two value that indicates the size of the palette for the gif.
        Defaults to 256.
    converter : str
        The converter to use. Right now, there are three converters:
        - ``'imagemagick'``
        - ``'imagemagick_tempfiles'``
        - ``'imageio'``

        The ``imagemagick[_tempfiles]`` converters are the
        only ones that support transparent gifs.

        Defaults to ``'imageio'``.
    converter_opts : dict
        Dictionary with options for the converters.
    """

    def __init__(self, filename, *args, **kwargs):
        super().__init__(filename, *args, **kwargs)
        self.color_count = kwargs.get("color_count", 256)
        self.converter = kwargs.get("converter", "imageio")
        self.converter_opts = kwargs.get("converter_opts", dict())

    def set_converter_opts(self, **kwargs):
        """Sets the options for the current converter."""
        self.converter_opts.update(kwargs)
        return self

    def save_with_imagemagick_tempfiles(self, frames, filename):
        """Writes this animation to a GIF file using ImageMagick, using temporary files.

        This exporter supports transparent backgrounds.
        This saves every frame to a temporary file.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """
        # this is stupid

        _filename, _file_extension = os.path.splitext(filename)

        temp_filenames = []

        index = 0

        for frame in frames:
            temp_name = "{}_TEMP_{:04d}.png".format(_filename, index)
            temp_filenames.append(temp_name)
            imageio.imsave(temp_name, frame)
            index += 1

        delay = int(100 / self.fps)
        fuzz = self.converter_opts.get("fuzz", 1)
        layer_opt = self.converter_opts.get("layer_opt", "OptimizeTransparency")

        cmd = [
            IMAGEMAGICK_BINARY,
            "-delay", str(delay),
            "-dispose", "{:d}".format(2 if self.converter_opts.get("dispose", False) or self.transparent else 1),
            "-loop", "{:d}".format(self.converter_opts.get("loop", 0)),
            "{}_TEMP_*.png".format(_filename),
            "-coalesce",
            "-layers", layer_opt,
            "-colors", str(self.color_count),
            "-fuzz", "{:02d}%".format(fuzz),
            filename
        ]

        proc = Popen(cmd)
        proc.wait()

        for f in temp_filenames:
            os.remove(f)

    def save_with_imagemagick(self, frames, filename):
        """Writes this animation to a GIF file using ImageMagick and FFmpeg.

        This exporter supports transparent backgrounds.
        This uses piping to avoid temporary files.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """

        # NOTE: main idea here is to grab frames using ffmpeg,
        # and pipe those to imagemagick's convert.
        # the reason we don't just use ffmpeg is because
        # it produces incredibly grainy gifs by default,
        # and most people on the web say that the best way is
        # to use imagemagick *shrugs*
        # this is based on moviepy's gif writers
        # (https://github.com/Zulko/moviepy/blob/master/moviepy/video/io/gif_writers.py)
        # thanks!

        delay = 100.0 / self.fps

        # NOTE: for the last -vcodec, ppm or bmp works, png doesn't???
        # ppm is fine for non transparent stuff,
        # otherwise bmp is the one that works properly.
        # see also: http://www.ffmpeg.org/general.html#toc-Image-Formats

        ffmpeg_command = [
            FFMPEG_BINARY,
            "-y",
            "-loglevel", "error",
            "-f", "rawvideo",
            "-vcodec", "rawvideo",
            "-r", "{:.02f}".format(self.fps),
            "-s", "{:d}x{:d}".format(self.w, self.h),
            "-pix_fmt", "rgba",
            "-i", "-",
            "-f", "image2pipe",
            "-vcodec", ("bmp" if self.transparent else "ppm"),
            "-"
        ]

        popen_kwargs = {"stdout": DEVNULL, "stderr": DEVNULL, "stdin": DEVNULL}

        # NOTE: CREATE_NO_WINDOW
        # see https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863%28v=vs.85%29.aspx
        if os.name == "nt":
            popen_kwargs["creationflags"] = 0x08000000

        popen_kwargs["stdin"] = PIPE
        popen_kwargs["stdout"] = PIPE

        ffmpeg_process = Popen(ffmpeg_command, **popen_kwargs)

        # RE: dispose
        # see http://www.imagemagick.org/script/command-line-options.php#dispose
        # we use 2 if the animation is meant to be transparent
        # otherwise you're on your own, and should set the "dispose" option
        # in the converter_opts arg in the constructor, or set_converter_opts
        im_command = [
            IMAGEMAGICK_BINARY,
            "-delay", "{:.02f}".format(delay),
            "-dispose", "{:d}".format(2 if self.converter_opts.get("dispose", False) or self.transparent else 1),
            "-loop", "{:d}".format(self.converter_opts.get("loop", 0)),
        ]

        im_command.extend(shlex.split(self.converter_opts.get("before_args", "")))
        im_command.extend(["-", "-coalesce", filename])

        popen_kwargs["stdin"] = ffmpeg_process.stdout
        popen_kwargs["stdout"] = DEVNULL
        im_process = Popen(im_command, **popen_kwargs)

        for frame in frames:
            ffmpeg_process.stdin.write(frame.tostring())

        ffmpeg_process.stdin.close()
        ffmpeg_process.wait()

        im_process.wait()

    def save_with_imageio(self, frames, filename):
        """Writes this animation to a GIF file using imageio.

        This exporter does not support transparent backgrounds.

        Parameters
        ----------
        frames : list of numpy arrays
            Container with the frames necessary to render this animation to a file.
        filename : str
            The filename to use when saving the file.
        """
        # TODO: make this more customizable

        quant = self.converter_opts.get("quantizer", "wu")

        writer = imageio.get_writer(
            filename,
            duration=1 / self.fps,
            quantizer=quant,
            palettesize=self.color_count
        )

        for frame in frames:
            writer.append_data(frame)

        writer.close()

    def render_and_save(self, filename=None):
        """Renders the animation and writes it to a GIF file.

        This uses the currently set exporter based on the attribute ``converter``.
        If ``converter`` is somehow ``None``, or some unknown value, the default
        is to export the animation using imageio.

        Parameters
        ----------
        filename : str
            The filename to use when saving the file.
        """
        if filename is None:
            filename = self.filename

        frames = self.render_all()
        func = getattr(self, "save_with_{}".format(self.converter), None)

        if func is None:
            self.save_with_imageio(frames, filename)
            return
        func(frames, filename)
<|endoftext|>


<commit_before>#
# Typofinder for domain typo discovery
#
# Released as open source by NCC Group Plc - http://www.nccgroup.com/
#
# Simple whois query function
#
# Based on RFC3912
#
# Developed by Matt Summers, matt dot summers at nccgroup dot com
#          and Stephen Tomkinson
#
# http://www.github.com/nccgroup/typofinder
#
# Released under AGPL see LICENSE for more information
#

import socket
import codecs
import re

from publicsuffix import PublicSuffixList
import datetime
import pprint
import sys

#Seed the whois server map with special cases that aren't in our whois-servers.txt list nor returned by iana
#Based on http://www.nirsoft.net/whois-servers.txt
FIELD_SEPERATOR = ', '
_tld_to_whois = dict()

with open("datasources/whois-servers.txt", "r") as whois_servers:
    for line in whois_servers:
        if line.startswith(';'):
            continue
        parts = line.split(' ')
        _tld_to_whois['.' + parts[0].strip()] = parts[1].strip()

_psl = PublicSuffixList(input_file=codecs.open("datasources/effective_tld_names.dat", "r", "utf8"))

def _whois_lookup(sServer, sDomain):
    """
    Perform the network connection to the Whois Server and query for the given domain.

    @param sServer: The hostname of the whois server to query.
    @param sDomain: The domain to query for.
    @return: The whois result string.
    """

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(20)
    s.connect((sServer, 43))
    try:
        query = str(codecs.encode(sDomain, "idna"), "ascii") + '\r\n'
    except:
        #Assumes an encoding error, just send the raw string instead.
        query = sDomain + '\r\n'

    response = ''

    try:
        s.send(query.encode())

        while len(response) < 10000:
            bytes = s.recv(1000)
            try:
                block = bytes.decode("utf-8")
            except:
                #If it's not UTF-8, the second most popular encoding appears to be iso-8859-1
                block = bytes.decode("iso-8859-1")

            if block == '':
                break
            response = response + block
    except socket.error:
        pass
    finally:
        s.shutdown(socket.SHUT_RDWR)
        s.close()

    return response

def whois(sDomain):
    """
    Entry point for this package, which fetches whois data from the appropriate server.

    @param sDomain: The domain to query whois for.
    @return: The whois result.
    """
    sDomain = _psl.get_public_suffix(sDomain)

    sLDot = sDomain.find(".")
    tld = sDomain[sLDot:]

    if tld in _tld_to_whois:
        sServer = _tld_to_whois[tld]
    else:
        sServer = "whois.iana.org"

        try:
            for sLine in _whois_lookup(sServer, tld).split('\n'):
                if "refer:" in sLine or "whois:" in sLine:
                    sServer = sLine[6:].lstrip()
                    _tld_to_whois[tld] = sServer
                    break
        except:
            pass

    result = _recursive_whois(sServer, sDomain)
    #Special case to handle the fuzzy matching at the ICANN whois server
    if 'To single out one record, look it up with "xxx", where xxx is one of the' in result:
        all_domain_records = _whois_lookup(sServer, '=' + sDomain)
        all_whois_servers = _extract_field(all_domain_records, "Whois Server")
        if all_whois_servers != None:
            next_whois_server = all_whois_servers.split(', ')[-1]
            return _recursive_whois(next_whois_server, sDomain)
        else:
            return result
    else:
        return result

def _recursive_whois(sServer, sDomain):
    """
    A recursive whois function which will follow the "Whois Server:" referals.

    @param sServer: The hostname of the whois server to query.
    @param sDomain: The domain to query for.
    @return: The whois result string.
    """
    result = _whois_lookup(sServer, sDomain)

    try:
        next_whois_server = _extract_field(result, "Whois Server")
        if next_whois_server and next_whois_server != sServer:
            return _recursive_whois(next_whois_server, sDomain)
    except:
        pass

    return result.lstrip()

def _extract_field(whois_blob, *args):
    """
    Extract from the given WHOIS result blob the value that is associated with the given field name.

    @param whois_blob The whois data to search for the value
    @param *args One or more field names (interpreted as regexes) that the requested value may be referred to as.
    """
    result = list()

    if len(args) == 1:
        field_name = args[0]
    else:
        field_name = "(?:"

        field_list = list()
        for arg in args:
            field_list.append("(?:" + arg + ")")

        field_name += "|".join(field_list)
        field_name += ")"

    regex = field_name + r"\.*:(?: |\t)*(.+)\n"

    match_list = re.finditer(regex, whois_blob, flags=re.IGNORECASE)

    for match in match_list:
        if match.group(1):
            value = match.group(1).strip()
            if value and value != "null":
                result.append(value)

    if not result:
        return None
    else:
        return FIELD_SEPERATOR.join(result)

def _date_parse(date_string):
    """
    Date parser which attempts to work with a range of date formats.

    @param date_string The string representing a date or date/time.
    @return A datetime object if one could be parsed, or None
    """

    if not date_string:
        return None
    date_string = date_string.rstrip('.')
    date_string = re.sub('(\d)T(\d)', '\g<1>\g<2>', date_string)
    date_string = date_string.replace(' ', '')
    date_string = date_string.replace('.', '-')
    date_string = date_string.rstrip('Z')

    #Handle timezones ourselves on python 2.X because the native datetime won't parse them
    tz_match = None
    if sys.version_info < (3,0):
        tz_match = re.match(r"(.*)(\+|-)(\d{2}):?(\d{2})$", date_string)
        if tz_match:
            date_string = tz_match.group(1)

    result = None

    for format in ("%Y-%m-%d%H:%M:%S", "%Y-%m-%d%H:%M:%S%z", "%Y-%m-%d", "%d-%b-%Y", "%a%b%d%H:%M:%S%Z%Y", "%Y-%d-%m"):
        try:
            result = datetime.datetime.strptime(date_string, format)
            break
        except ValueError:
            #Attempt the next format
            continue

    if result and tz_match:
        #Manipulate the datetime into UTC if we don't have timezone support
        delta = datetime.timedelta(hours=int(tz_match.group(3)), minutes=int(tz_match.group(4)))
        if tz_match.group(2) == '-':
            result += delta
        else:
            result -= delta

    return result

contact_types = {"registrant": "Registrant(?: Contact)?",
                 "tech": "Tech(?:nical)?(?: Contact)?",
                 "admin": "Admin(?:istrative)?(?: Contact)?"}

contact_fields = {"name": "(?:Name)?",
                  "organization": "Organization",
                  "street": "(?:(?:Street)|(?:Address ?1))",
                  "city": "City",
                  "state": "State/Province",
                  "country": "Country",
                  "post_code": "Postal ?Code",
                  "email": "E-?mail",
                  "phone": "Phone(?: Number)?",
                  "phone_ext": "Phone Ext",
                  "fax": "Fax(?: Number)?",
                  "fax_ext": "Fax Ext"}

registrar_fields = {"name": "Registrar(?: Name)?",
                    "url": "Registrar (?:(?:URL)|(?:Homepage))",
                    "abuse_email": "Abuse Contact Email",
                    "abuse_phone": "Abuse Contact Phone",
                    "iana_id": "Registrar IANA ID"}

date_fields = {"created": ("Creation Date", "created", "Registered(?: on)?", "Created(?: on)?", "Registration Date"),
               "updated": ("Last Modified", "Updated Date", "(?:last )?updated?(?: on)?"),
               "expires": ("Expiration Date", "Expiry Date", "renewal date", "Expires(?: on)?", "Expire Date")}

def parse(whois_str):
    """
    Parses the given whois result string in an attempt to extract common fields.

    @param whois_str The raw WHOIS result
    @return A dictionary of dictionaries containing the parsed data.
    """
    result_dict = {}

    for type in contact_types.keys():
        person_dict = dict()

        for field in contact_fields.keys():
            person_dict[field] = _extract_field(whois_str, contact_types[type] + " " + contact_fields[field])

        result_dict[type] = person_dict

    registrar_dict = dict()
    for field in registrar_fields.keys():
        registrar_dict[field] = _extract_field(whois_str, registrar_fields[field])

    result_dict['registrar'] = registrar_dict

    result_dict['reseller'] = {'name': _extract_field(whois_str, "Reseller")}

    dates_dict = {}
    for field in date_fields.keys():
        date_str = _extract_field(whois_str, *date_fields[field])
        if date_str:
            date_str = date_str.split(FIELD_SEPERATOR)[0]
            dates_dict[field] = _date_parse(date_str)
        else:
            dates_dict[field] = None

    result_dict['date'] = dates_dict

    return result_dict<commit_msg>Improve robustness of socket shutdown.<commit_after>#
# Typofinder for domain typo discovery
#
# Released as open source by NCC Group Plc - http://www.nccgroup.com/
#
# Simple whois query function
#
# Based on RFC3912
#
# Developed by Matt Summers, matt dot summers at nccgroup dot com
#          and Stephen Tomkinson
#
# http://www.github.com/nccgroup/typofinder
#
# Released under AGPL see LICENSE for more information
#

import socket
import codecs
import re

from publicsuffix import PublicSuffixList
import datetime
import pprint
import sys

#Seed the whois server map with special cases that aren't in our whois-servers.txt list nor returned by iana
#Based on http://www.nirsoft.net/whois-servers.txt
FIELD_SEPERATOR = ', '
_tld_to_whois = dict()

with open("datasources/whois-servers.txt", "r") as whois_servers:
    for line in whois_servers:
        if line.startswith(';'):
            continue
        parts = line.split(' ')
        _tld_to_whois['.' + parts[0].strip()] = parts[1].strip()

_psl = PublicSuffixList(input_file=codecs.open("datasources/effective_tld_names.dat", "r", "utf8"))

def _whois_lookup(sServer, sDomain):
    """
    Perform the network connection to the Whois Server and query for the given domain.

    @param sServer: The hostname of the whois server to query.
    @param sDomain: The domain to query for.
    @return: The whois result string.
    """

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(20)
    s.connect((sServer, 43))
    try:
        query = str(codecs.encode(sDomain, "idna"), "ascii") + '\r\n'
    except:
        #Assumes an encoding error, just send the raw string instead.
        query = sDomain + '\r\n'

    response = ''

    try:
        s.send(query.encode())

        while len(response) < 10000:
            bytes = s.recv(1000)
            try:
                block = bytes.decode("utf-8")
            except:
                #If it's not UTF-8, the second most popular encoding appears to be iso-8859-1
                block = bytes.decode("iso-8859-1")

            if block == '':
                break
            response = response + block
    except socket.error:
        pass
    finally:
        try:
            s.shutdown(socket.SHUT_RDWR)
        except socket.error:
            #Not much more we can do here
            pass
        finally:
            s.close()

    return response

def whois(sDomain):
    """
    Entry point for this package, which fetches whois data from the appropriate server.

    @param sDomain: The domain to query whois for.
    @return: The whois result.
    """
    sDomain = _psl.get_public_suffix(sDomain)

    sLDot = sDomain.find(".")
    tld = sDomain[sLDot:]

    if tld in _tld_to_whois:
        sServer = _tld_to_whois[tld]
    else:
        sServer = "whois.iana.org"

        try:
            for sLine in _whois_lookup(sServer, tld).split('\n'):
                if "refer:" in sLine or "whois:" in sLine:
                    sServer = sLine[6:].lstrip()
                    _tld_to_whois[tld] = sServer
                    break
        except:
            pass

    result = _recursive_whois(sServer, sDomain)
    #Special case to handle the fuzzy matching at the ICANN whois server
    if 'To single out one record, look it up with "xxx", where xxx is one of the' in result:
        all_domain_records = _whois_lookup(sServer, '=' + sDomain)
        all_whois_servers = _extract_field(all_domain_records, "Whois Server")
        if all_whois_servers != None:
            next_whois_server = all_whois_servers.split(', ')[-1]
            return _recursive_whois(next_whois_server, sDomain)
        else:
            return result
    else:
        return result

def _recursive_whois(sServer, sDomain):
    """
    A recursive whois function which will follow the "Whois Server:" referals.

    @param sServer: The hostname of the whois server to query.
    @param sDomain: The domain to query for.
    @return: The whois result string.
    """
    result = _whois_lookup(sServer, sDomain)

    try:
        next_whois_server = _extract_field(result, "Whois Server")
        if next_whois_server and next_whois_server != sServer:
            return _recursive_whois(next_whois_server, sDomain)
    except:
        pass

    return result.lstrip()

def _extract_field(whois_blob, *args):
    """
    Extract from the given WHOIS result blob the value that is associated with the given field name.

    @param whois_blob The whois data to search for the value
    @param *args One or more field names (interpreted as regexes) that the requested value may be referred to as.
    """
    result = list()

    if len(args) == 1:
        field_name = args[0]
    else:
        field_name = "(?:"

        field_list = list()
        for arg in args:
            field_list.append("(?:" + arg + ")")

        field_name += "|".join(field_list)
        field_name += ")"

    regex = field_name + r"\.*:(?: |\t)*(.+)\n"

    match_list = re.finditer(regex, whois_blob, flags=re.IGNORECASE)

    for match in match_list:
        if match.group(1):
            value = match.group(1).strip()
            if value and value != "null":
                result.append(value)

    if not result:
        return None
    else:
        return FIELD_SEPERATOR.join(result)

def _date_parse(date_string):
    """
    Date parser which attempts to work with a range of date formats.

    @param date_string The string representing a date or date/time.
    @return A datetime object if one could be parsed, or None
    """

    if not date_string:
        return None
    date_string = date_string.rstrip('.')
    date_string = re.sub('(\d)T(\d)', '\g<1>\g<2>', date_string)
    date_string = date_string.replace(' ', '')
    date_string = date_string.replace('.', '-')
    date_string = date_string.rstrip('Z')

    #Handle timezones ourselves on python 2.X because the native datetime won't parse them
    tz_match = None
    if sys.version_info < (3,0):
        tz_match = re.match(r"(.*)(\+|-)(\d{2}):?(\d{2})$", date_string)
        if tz_match:
            date_string = tz_match.group(1)

    result = None

    for format in ("%Y-%m-%d%H:%M:%S", "%Y-%m-%d%H:%M:%S%z", "%Y-%m-%d", "%d-%b-%Y", "%a%b%d%H:%M:%S%Z%Y", "%Y-%d-%m"):
        try:
            result = datetime.datetime.strptime(date_string, format)
            break
        except ValueError:
            #Attempt the next format
            continue

    if result and tz_match:
        #Manipulate the datetime into UTC if we don't have timezone support
        delta = datetime.timedelta(hours=int(tz_match.group(3)), minutes=int(tz_match.group(4)))
        if tz_match.group(2) == '-':
            result += delta
        else:
            result -= delta

    return result

contact_types = {"registrant": "Registrant(?: Contact)?",
                 "tech": "Tech(?:nical)?(?: Contact)?",
                 "admin": "Admin(?:istrative)?(?: Contact)?"}

contact_fields = {"name": "(?:Name)?",
                  "organization": "Organization",
                  "street": "(?:(?:Street)|(?:Address ?1))",
                  "city": "City",
                  "state": "State/Province",
                  "country": "Country",
                  "post_code": "Postal ?Code",
                  "email": "E-?mail",
                  "phone": "Phone(?: Number)?",
                  "phone_ext": "Phone Ext",
                  "fax": "Fax(?: Number)?",
                  "fax_ext": "Fax Ext"}

registrar_fields = {"name": "Registrar(?: Name)?",
                    "url": "Registrar (?:(?:URL)|(?:Homepage))",
                    "abuse_email": "Abuse Contact Email",
                    "abuse_phone": "Abuse Contact Phone",
                    "iana_id": "Registrar IANA ID"}

date_fields = {"created": ("Creation Date", "created", "Registered(?: on)?", "Created(?: on)?", "Registration Date"),
               "updated": ("Last Modified", "Updated Date", "(?:last )?updated?(?: on)?"),
               "expires": ("Expiration Date", "Expiry Date", "renewal date", "Expires(?: on)?", "Expire Date")}

def parse(whois_str):
    """
    Parses the given whois result string in an attempt to extract common fields.

    @param whois_str The raw WHOIS result
    @return A dictionary of dictionaries containing the parsed data.
    """
    result_dict = {}

    for type in contact_types.keys():
        person_dict = dict()

        for field in contact_fields.keys():
            person_dict[field] = _extract_field(whois_str, contact_types[type] + " " + contact_fields[field])

        result_dict[type] = person_dict

    registrar_dict = dict()
    for field in registrar_fields.keys():
        registrar_dict[field] = _extract_field(whois_str, registrar_fields[field])

    result_dict['registrar'] = registrar_dict

    result_dict['reseller'] = {'name': _extract_field(whois_str, "Reseller")}

    dates_dict = {}
    for field in date_fields.keys():
        date_str = _extract_field(whois_str, *date_fields[field])
        if date_str:
            date_str = date_str.split(FIELD_SEPERATOR)[0]
            dates_dict[field] = _date_parse(date_str)
        else:
            dates_dict[field] = None

    result_dict['date'] = dates_dict

    return result_dict<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
from time import sleep
from fabric.api import *
from fabric.contrib.console import confirm
from fabric.contrib.files import exists
from fabric.utils import abort

# Globals

env.project_name = 'editorsnotes'

# Environments

def dev():
    "Use the development webserver."
    env.hosts = ['editorsnotes.local']
    env.user = 'ryanshaw'
    env.path = '/Library/WebServer/Documents/%(project_name)s' % env
    env.vhosts_path = '/etc/apache2/sites'
    env.site_packages = '/Library/Python/2.6/site-packages'

def pro():
    "Use the production webserver."
    env.hosts = ['editorsnotes.org']
    env.user = 'ryanshaw'
    env.path = '/db/projects/%(project_name)s' % env
    env.vhosts_path = '/etc/httpd/sites.d'
    env.site_packages = '/usr/lib64/python2.6/site-packages'

# Tasks

def test():
    "Run the test suite locally."
    local("cd %(project_name)s; python manage.py test" % env)

def test_remote():
    "Run the test suite remotely."
    require('hosts', provided_by=[dev])
    require('path')
    run('cd %(path)s/releases/current/%(project_name)s;  ../../../bin/python manage.py test' % env)

def setup():
    """
    Setup a fresh virtualenv as well as a few useful directories, then
    run a full deployment.
    """
    require('hosts', provided_by=[dev])
    require('path')
    run('mkdir -p %(path)s' % env)
    with cd(env.path):
        run('virtualenv --no-site-packages .' % env)
        run('mkdir -p logs; mkdir -p releases; mkdir -p shared; mkdir -p packages' % env)
        run('cd releases; touch none; ln -sf none current; ln -sf none previous')
    deploy()

def deploy():
    """
    Deploy the latest version of the site to the servers, install any
    required third party modules, install the virtual host and then
    restart the webserver.
    """
    require('hosts', provided_by=[dev])
    require('path')
    import time
    env.release = time.strftime('%Y%m%d%H%M%S')
    upload_tar_from_git()
    upload_local_settings()
    symlink_system_packages()
    install_requirements()
    install_site()
    symlink_current_release()
    migrate()
    restart_webserver()
    sleep(2)
    local('open http://%(host)s/' % env)

def deploy_version(version):
    "Specify a specific version to be made live."
    require('hosts', provided_by=[dev])
    require('path')
    env.version = version
    with cd(env.path):
        run('rm releases/previous; mv releases/current releases/previous')
        run('ln -s %(version)s releases/current' % env)
    restart_webserver()

def rollback():
    """
    Limited rollback capability. Simple loads the previously current
    version of the code. Rolling back again will swap between the two.
    """
    require('hosts', provided_by=[dev])
    require('path')
    with cd(env.path):
        run('mv releases/current releases/_previous;')
        run('mv releases/previous releases/current;')
        run('mv releases/_previous releases/previous;')
    restart_webserver()

def clean():
    "Clean out old packages and releases."
    require('hosts', provided_by=[dev])
    require('path')
    if (confirm('Are you sure you want to delete everything on %(host)s?' % env,
                default=False)):
        with cd(env.path):
            run('rm -rf packages; rm -rf releases')
            run('mkdir -p packages; mkdir -p releases')
            run('cd releases; touch none; ln -sf none current; ln -sf none previous')

# Helpers. These are called by other functions rather than directly.

def upload_tar_from_git():
    "Create an archive from the current Git master branch and upload it."
    require('release', provided_by=[deploy, setup])
    local('git archive --format=tar master | gzip > %(release)s.tar.gz' % env)
    run('mkdir -p %(path)s/releases/%(release)s' % env)
    put('%(release)s.tar.gz' % env, '%(path)s/packages/' % env)
    run('cd %(path)s/releases/%(release)s && tar zxf ../../packages/%(release)s.tar.gz' % env)
    local('rm %(release)s.tar.gz' % env)

def upload_local_settings():
    "Upload the appropriate local settings file."
    require('release', provided_by=[deploy, setup])
    put('%(project_name)s/settings-%(host)s.py' % env,
        '%(path)s/releases/%(release)s/%(project_name)s/settings_local.py' % env)

def install_requirements():
    "Install the required packages from the requirements file using pip"
    require('release', provided_by=[deploy, setup])
    run('export SAVED_PIP_VIRTUALENV_BASE=$PIP_VIRTUALENV_BASE; unset PIP_VIRTUALENV_BASE; ' +
        'cd %(path)s; ./bin/pip install -E . -r ./releases/%(release)s/requirements.txt; ' % env +
        'export PIP_VIRTUALENV_BASE=$SAVED_PIP_VIRTUALENV_BASE; unset SAVED_PIP_VIRTUALENV_BASE')

def symlink_system_packages():
    "Create symlinks to system site-packages."
    require('site_packages', provided_by=[dev])
    require('path')
    site_packages = env.path + '/lib/python2.6/site-packages'
    with cd(site_packages):
        with open('requirements.txt') as reqs:
            for line in reqs:
                if line.startswith('# symlink: '):
                    target = env.site_packages + '/' + line[11:-1]
                    if exists(target):
                        run('ln -f -s %s' % target)
                    else:
                        abort('Missing %s' % target)

def install_site():
    "Add the virtualhost file to apache."
    require('release', provided_by=[deploy, setup])
    sudo('cd %(path)s/releases/%(release)s; cp -f vhost-%(host)s.conf %(vhosts_path)s' % env, pty=True)

def symlink_current_release():
    "Symlink our current release."
    require('release', provided_by=[deploy, setup])
    with cd(env.path):
        run('rm releases/previous; mv releases/current releases/previous;')
        run('ln -s %(release)s releases/current' % env)

def migrate():
    "Update the database"
    require('hosts', provided_by=[dev])
    require('path')
    with cd('%(path)s/releases/current/%(project_name)s' % env):
        run('../../../bin/python manage.py syncdb --noinput' % env)
        #run('../../../bin/python manage.py migrate main')
        #run('../../../bin/python manage.py rebuild_index')

def restart_webserver():
    "Restart the web server."
    sudo('apachectl restart', pty=True)
<commit_msg>Re-add migrattion command.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
from time import sleep
from fabric.api import *
from fabric.contrib.console import confirm
from fabric.contrib.files import exists
from fabric.utils import abort

# Globals

env.project_name = 'editorsnotes'

# Environments

def dev():
    "Use the development webserver."
    env.hosts = ['editorsnotes.local']
    env.user = 'ryanshaw'
    env.path = '/Library/WebServer/Documents/%(project_name)s' % env
    env.vhosts_path = '/etc/apache2/sites'
    env.site_packages = '/Library/Python/2.6/site-packages'

def pro():
    "Use the production webserver."
    env.hosts = ['editorsnotes.org']
    env.user = 'ryanshaw'
    env.path = '/db/projects/%(project_name)s' % env
    env.vhosts_path = '/etc/httpd/sites.d'
    env.site_packages = '/usr/lib64/python2.6/site-packages'

# Tasks

def test():
    "Run the test suite locally."
    local("cd %(project_name)s; python manage.py test" % env)

def test_remote():
    "Run the test suite remotely."
    require('hosts', provided_by=[dev])
    require('path')
    run('cd %(path)s/releases/current/%(project_name)s;  ../../../bin/python manage.py test' % env)

def setup():
    """
    Setup a fresh virtualenv as well as a few useful directories, then
    run a full deployment.
    """
    require('hosts', provided_by=[dev])
    require('path')
    run('mkdir -p %(path)s' % env)
    with cd(env.path):
        run('virtualenv --no-site-packages .' % env)
        run('mkdir -p logs; mkdir -p releases; mkdir -p shared; mkdir -p packages' % env)
        run('cd releases; touch none; ln -sf none current; ln -sf none previous')
    deploy()

def deploy():
    """
    Deploy the latest version of the site to the servers, install any
    required third party modules, install the virtual host and then
    restart the webserver.
    """
    require('hosts', provided_by=[dev])
    require('path')
    import time
    env.release = time.strftime('%Y%m%d%H%M%S')
    upload_tar_from_git()
    upload_local_settings()
    symlink_system_packages()
    install_requirements()
    install_site()
    symlink_current_release()
    migrate()
    restart_webserver()
    sleep(2)
    local('open http://%(host)s/' % env)

def deploy_version(version):
    "Specify a specific version to be made live."
    require('hosts', provided_by=[dev])
    require('path')
    env.version = version
    with cd(env.path):
        run('rm releases/previous; mv releases/current releases/previous')
        run('ln -s %(version)s releases/current' % env)
    restart_webserver()

def rollback():
    """
    Limited rollback capability. Simple loads the previously current
    version of the code. Rolling back again will swap between the two.
    """
    require('hosts', provided_by=[dev])
    require('path')
    with cd(env.path):
        run('mv releases/current releases/_previous;')
        run('mv releases/previous releases/current;')
        run('mv releases/_previous releases/previous;')
    restart_webserver()

def clean():
    "Clean out old packages and releases."
    require('hosts', provided_by=[dev])
    require('path')
    if (confirm('Are you sure you want to delete everything on %(host)s?' % env,
                default=False)):
        with cd(env.path):
            run('rm -rf packages; rm -rf releases')
            run('mkdir -p packages; mkdir -p releases')
            run('cd releases; touch none; ln -sf none current; ln -sf none previous')

# Helpers. These are called by other functions rather than directly.

def upload_tar_from_git():
    "Create an archive from the current Git master branch and upload it."
    require('release', provided_by=[deploy, setup])
    local('git archive --format=tar master | gzip > %(release)s.tar.gz' % env)
    run('mkdir -p %(path)s/releases/%(release)s' % env)
    put('%(release)s.tar.gz' % env, '%(path)s/packages/' % env)
    run('cd %(path)s/releases/%(release)s && tar zxf ../../packages/%(release)s.tar.gz' % env)
    local('rm %(release)s.tar.gz' % env)

def upload_local_settings():
    "Upload the appropriate local settings file."
    require('release', provided_by=[deploy, setup])
    put('%(project_name)s/settings-%(host)s.py' % env,
        '%(path)s/releases/%(release)s/%(project_name)s/settings_local.py' % env)

def install_requirements():
    "Install the required packages from the requirements file using pip"
    require('release', provided_by=[deploy, setup])
    run('export SAVED_PIP_VIRTUALENV_BASE=$PIP_VIRTUALENV_BASE; unset PIP_VIRTUALENV_BASE; ' +
        'cd %(path)s; ./bin/pip install -E . -r ./releases/%(release)s/requirements.txt; ' % env +
        'export PIP_VIRTUALENV_BASE=$SAVED_PIP_VIRTUALENV_BASE; unset SAVED_PIP_VIRTUALENV_BASE')

def symlink_system_packages():
    "Create symlinks to system site-packages."
    require('site_packages', provided_by=[dev])
    require('path')
    site_packages = env.path + '/lib/python2.6/site-packages'
    with cd(site_packages):
        with open('requirements.txt') as reqs:
            for line in reqs:
                if line.startswith('# symlink: '):
                    target = env.site_packages + '/' + line[11:-1]
                    if exists(target):
                        run('ln -f -s %s' % target)
                    else:
                        abort('Missing %s' % target)

def install_site():
    "Add the virtualhost file to apache."
    require('release', provided_by=[deploy, setup])
    sudo('cd %(path)s/releases/%(release)s; cp -f vhost-%(host)s.conf %(vhosts_path)s' % env, pty=True)

def symlink_current_release():
    "Symlink our current release."
    require('release', provided_by=[deploy, setup])
    with cd(env.path):
        run('rm releases/previous; mv releases/current releases/previous;')
        run('ln -s %(release)s releases/current' % env)

def migrate():
    "Update the database"
    require('hosts', provided_by=[dev])
    require('path')
    with cd('%(path)s/releases/current/%(project_name)s' % env):
        run('../../../bin/python manage.py syncdb --noinput')
        run('../../../bin/python manage.py migrate --noinput main')
        #run('../../../bin/python manage.py rebuild_index')

def restart_webserver():
    "Restart the web server."
    sudo('apachectl restart', pty=True)
<|endoftext|>


<commit_before>import os
import base64
from datetime import datetime
from planetstack.config import Config
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q

logger = Logger(level=logging.INFO)

class FailedDependency(Exception):
    pass

class SyncStep:
    """ A PlanetStack Sync step.

    Attributes:
        psmodel        Model name the step synchronizes
        dependencies    list of names of models that must be synchronized first if the current model depends on them
    """
    slow=False
    def get_prop(prop):
        try:
            sync_config_dir = Config().sync_config_dir
        except:
            sync_config_dir = '/etc/planetstack/sync'
        prop_config_path = '/'.join(sync_config_dir,self.name,prop)
        return open(prop_config_path).read().rstrip()

    def __init__(self, **args):
        """Initialize a sync step
           Keyword arguments:
                   name -- Name of the step
                provides -- PlanetStack models sync'd by this step
        """
        dependencies = []
        self.driver = args.get('driver')
        self.error_map = args.get('error_map')

        try:
            self.soft_deadline = int(self.get_prop('soft_deadline_seconds'))
        except:
            self.soft_deadline = 5 # 5 seconds

        return

    def fetch_pending(self, deletion=False):
        # This is the most common implementation of fetch_pending
        # Steps should override it if they have their own logic
        # for figuring out what objects are outstanding.
        main_obj = self.provides[0]
        if (not deletion):
            objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
        else:
            objs = main_obj.deleted_objects.all()

        return objs
        #return Sliver.objects.filter(ip=None)

    def check_dependencies(self, obj, failed):
        for dep in self.dependencies:
            peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
            peer_object = getattr(obj, peer_name)
            if (peer_object and peer_object.pk==failed.pk and type(peer_object)==type(failed)):
                raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed  %s:%s" % (obj.__class__.__name__, str(obj.pk\
), peer_object.__class__.__name__, str(peer_object.pk), failed.__class__.__name__, str(failed.pk)))

    def call(self, failed=[], deletion=False):
        pending = self.fetch_pending(deletion)
        for o in pending:
            try:
                for f in failed:
                    self.check_dependencies(o,f) # Raises exception if failed
                if (deletion):
                    self.delete_record(o)
                    o.delete(purge=True)
                else:
                    self.sync_record(o)
                    o.enacted = datetime.now() # Is this the same timezone? XXX
                    o.backend_status = "OK"
                    o.save(update_fields=['enacted'])
            except Exception,e:
                try:
                    o.backend_status = self.error_map.map(str(e))
                except:
                    o.backend_status = str(e)

                if (o.pk):
                    o.save(update_fields=['backend_status'])

                logger.log_exc("sync step failed!")
                failed.append(o)

        return failed

    def __call__(self, **args):
        return self.call(**args)
<commit_msg>Bug fix from Scott Baker. If a peer object is not set, it means there is no dependency.<commit_after>import os
import base64
from datetime import datetime
from planetstack.config import Config
from util.logger import Logger, logging
from observer.steps import *
from django.db.models import F, Q

logger = Logger(level=logging.INFO)

class FailedDependency(Exception):
    pass

class SyncStep:
    """ A PlanetStack Sync step.

    Attributes:
        psmodel        Model name the step synchronizes
        dependencies    list of names of models that must be synchronized first if the current model depends on them
    """
    slow=False
    def get_prop(prop):
        try:
            sync_config_dir = Config().sync_config_dir
        except:
            sync_config_dir = '/etc/planetstack/sync'
        prop_config_path = '/'.join(sync_config_dir,self.name,prop)
        return open(prop_config_path).read().rstrip()

    def __init__(self, **args):
        """Initialize a sync step
           Keyword arguments:
                   name -- Name of the step
                provides -- PlanetStack models sync'd by this step
        """
        dependencies = []
        self.driver = args.get('driver')
        self.error_map = args.get('error_map')

        try:
            self.soft_deadline = int(self.get_prop('soft_deadline_seconds'))
        except:
            self.soft_deadline = 5 # 5 seconds

        return

    def fetch_pending(self, deletion=False):
        # This is the most common implementation of fetch_pending
        # Steps should override it if they have their own logic
        # for figuring out what objects are outstanding.
        main_obj = self.provides[0]
        if (not deletion):
            objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
        else:
            objs = main_obj.deleted_objects.all()

        return objs
        #return Sliver.objects.filter(ip=None)

    def check_dependencies(self, obj, failed):
        for dep in self.dependencies:
            peer_name = dep[0].lower() + dep[1:]    # django names are camelCased with the first letter lower
            try:
                peer_object = getattr(obj, peer_name)
            except:
                peer_object = None

            if (peer_object and peer_object.pk==failed.pk and type(peer_object)==type(failed)):
                raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed  %s:%s" % (obj.__class__.__name__, str(obj.pk\
), peer_object.__class__.__name__, str(peer_object.pk), failed.__class__.__name__, str(failed.pk)))

    def call(self, failed=[], deletion=False):
        pending = self.fetch_pending(deletion)
        for o in pending:
            try:
                for f in failed:
                    self.check_dependencies(o,f) # Raises exception if failed
                if (deletion):
                    self.delete_record(o)
                    o.delete(purge=True)
                else:
                    self.sync_record(o)
                    o.enacted = datetime.now() # Is this the same timezone? XXX
                    o.backend_status = "OK"
                    o.save(update_fields=['enacted'])
            except Exception,e:
                try:
                    o.backend_status = self.error_map.map(str(e))
                except:
                    o.backend_status = str(e)

                if (o.pk):
                    o.save(update_fields=['backend_status'])

                logger.log_exc("sync step failed!")
                failed.append(o)

        return failed

    def __call__(self, **args):
        return self.call(**args)
<|endoftext|>


<commit_before>from fabric.api import local, lcd, run, cd, settings
import os

def update_repository(commit_args, directory="."):

    with lcd(directory):
        # with settings(warn_only=True):
        if local("git status --porcelain", capture=True):
            local("git add . ")
            local("git commit {}".format(commit_args))
            local("git push origin master")

def prepare(commit_args=""):

    # TODO: add testing
    update_repository(commit_args, directory="../swc_blog")

    local('python3 manage.py test')
    update_repository(commit_args)

def stage(project_dir, commit_args=""):
    prepare(commit_args)

    with settings(warn_only=True):
        if run("test -d {}".format(project_dir)).failed:
            run("sudo mkdir {}".format(project_dir))
            run("sudo git clone https://github.com/e9wikner/swc_site.git {}".format(
                project_dir))
            run("sudo chown -R http {}".format(project_dir))
            run("sudo chgrp -R http {}".format(project_dir))

    # Install blog app
    run("sudo pip install --upgrade git+https://github.com/e9wikner/swc_blog")

    with cd(project_dir):
        run("sudo git pull")
        run("sudo python3 manage.py makemigrations")
        run("sudo python3 manage.py migrate")

def deploy():
    prepare()

    with lcd('/path/to/my/prod/area/'):

        # With git...
        local('git pull /my/path/to/dev/area/')

        # With both
        local('python manage.py migrate myapp')
        local('python manage.py test myapp')
        local('/my/command/to/restart/webserver')<commit_msg>Fixed<commit_after>from fabric.api import local, lcd, run, cd, settings
import os

def update_repository(commit_args, directory="."):

    with lcd(directory):
        # with settings(warn_only=True):
        if local("git status --porcelain", capture=True):
            local("git add . ")
            local("git commit {}".format(commit_args))
            local("git push origin master")

def prepare(commit_args=""):

    # TODO: add testing
    update_repository(commit_args, directory="../swc_blog")

    local('python3 manage.py test')

    update_repository(commit_args)

def stage(project_dir, commit_args=""):
    prepare(commit_args)

    with settings(warn_only=True):
        if run("test -d {}".format(project_dir)).failed:
            run("sudo mkdir {}".format(project_dir))
            run("sudo git clone https://github.com/e9wikner/swc_site.git {}".format(
                project_dir))
            run("sudo chown -R http {}".format(project_dir))
            run("sudo chgrp -R http {}".format(project_dir))

    # Install blog app
    run("sudo pip install --upgrade git+https://github.com/e9wikner/swc_blog")

    with cd(project_dir):
        run("sudo git pull")
        run("sudo python3 manage.py makemigrations "
            "--settings=swc_site.settings.staging")
        run("sudo python3 manage.py migrate "
            "--settings=swc_site.settings.staging")

def deploy():
    prepare()

    with lcd('/path/to/my/prod/area/'):

        # With git...
        local('git pull /my/path/to/dev/area/')

        # With both
        local('python manage.py migrate myapp')
        local('python manage.py test myapp')
        local('/my/command/to/restart/webserver')<|endoftext|>