<commit_before>from __future__ import division
import numpy as np

#TODO: document me
def regression(X, T, regression_type, **kwargs):
    r"""
    Applies regression.
    """
    if hasattr(regression_type, '__call__'):
        regression_closure = regression_type(X, T, **kwargs)

        # note that, self is required as this closure is assigned to an object
        def regression_object_method(x):
            return regression_closure(x)

        return regression_closure
    else:
        raise ValueError("regression_type can only be: a closure defining "
                         "a particular regression technique. Several examples"
                         " of such closures can be found in "
                         "`menpo.sdm.functions` (mlr, pcr, pls, ccr, ...).")

def mlr(X, T):
    r"""
    Multivariate Linear Regression
    """
    XX = np.dot(X.T, X)
    XX = (XX + XX.T) / 2
    XT = np.dot(X.T, T)
    R = np.linalg.solve(XX, XT)

    def mlr_fitting(x):
        return np.dot(x, R)

    return mlr_fitting

def mlr_svd(X, T, variance=None):
    r"""
    Multivariate Linear Regression using SVD decomposition
    """
    R, _, _, _ = _svd_regression(X, T, variance=variance)

    def mlr_svd_fitting(x):
        return np.dot(x, R)

    return mlr_svd_fitting

def mlr_pca(X, T, variance=None):
    r"""
    Multivariate Linear Regression using PCA reconstructions
    """
    R, _, _, V = _svd_regression(X, T, variance=variance)

    def mlr_svd_fitting(x):
        x = np.dot(np.dot(x, V.T), V)
        return np.dot(x, R)

    return mlr_svd_fitting

def mlr_pca_weights(X, T, variance=None):
    r"""
    Multivariate Linear Regression using PCA weights
    """
    R, _, _, V = _svd_regression(X, T, variance=variance)
    W = np.dot(X, V.T)
    R, _, _, _ = _svd_regression(W, T)

    def mlr_svd_fitting(x):
        w = np.dot(x, V.T)
        return np.dot(w, R)

    return mlr_svd_fitting

def _svd_regression(X, T, variance=None):
    if variance is not None and not (0 < variance <= 1):
        raise ValueError("variance must be set to a number between 0 and 1.")

    U, s, V = np.linalg.svd(X)
    if variance:
        total = sum(s)
        acc = 0
        for j, y in enumerate(s):
            acc += y
            if acc / total >= variance:
                r = j+1
                break
    else:
        tol = np.max(X.shape) * np.spacing(np.max(s))
        r = np.sum(s > tol)
    U = U[:, :r]
    s = 1 / s[:r]
    V = V[:r, :]
    R = np.dot(np.dot(V.T * s, U.T), T)

    return R, U, s, V
<commit_msg>adds doc in regressionfunctions.py<commit_after>from __future__ import division
import numpy as np

def regression(X, T, regression_type, **kwargs):
    r"""
    Method that solves the regression equation.

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.
    regression_type: function/closure
        The closure of the regression method to be used. It must be one of
        the functions defined below.

    Returns
    -------
    regression_closure: function/closure
        The closure of the regression method.
    """
    if hasattr(regression_type, '__call__'):
        regression_closure = regression_type(X, T, **kwargs)

        # note that, self is required as this closure is assigned to an object
        def regression_object_method(x):
            return regression_closure(x)

        return regression_closure
    else:
        raise ValueError("regression_type can only be: a closure defining "
                         "a particular regression technique. Several examples"
                         " of such closures can be found in "
                         "`menpo.fit.regression.regressionfunctions.py` "
                         "(mlr, mlr_svd, mlr_pca, pcr, pls, ccr, ...).")

def mlr(X, T):
    r"""
    Multivariate Linear Regression

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.

    Returns
    -------
    mlr_fitting: function/closure
        The closure of the regression method.
    """
    XX = np.dot(X.T, X)
    XX = (XX + XX.T) / 2
    XT = np.dot(X.T, T)
    R = np.linalg.solve(XX, XT)

    def mlr_fitting(x):
        return np.dot(x, R)

    return mlr_fitting

def mlr_svd(X, T, variance=None):
    r"""
    Multivariate Linear Regression using SVD decomposition

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.
    variance: float or None, Optional
        The SVD variance.

        Default: None

    Returns
    -------
    mlr_svd_fitting: function/closure
        The closure of the regression method.

    Raises
    ------
    ValueError
        variance must be set to a number between 0 and 1
    """
    R, _, _, _ = _svd_regression(X, T, variance=variance)

    def mlr_svd_fitting(x):
        return np.dot(x, R)

    return mlr_svd_fitting

def mlr_pca(X, T, variance=None):
    r"""
    Multivariate Linear Regression using PCA reconstructions

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.
    variance: float or None, Optional
        The SVD variance.

        Default: None

    Returns
    -------
    mlr_pca_fitting: function/closure
        The closure of the regression method.

    Raises
    ------
    ValueError
        variance must be set to a number between 0 and 1
    """
    R, _, _, V = _svd_regression(X, T, variance=variance)

    def mlr_pca_fitting(x):
        x = np.dot(np.dot(x, V.T), V)
        return np.dot(x, R)

    return mlr_pca_fitting

def mlr_pca_weights(X, T, variance=None):
    r"""
    Multivariate Linear Regression using PCA weights

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.
    variance: float or None, Optional
        The SVD variance.

        Default: None

    Returns
    -------
    mlr_pca_weights_fitting: function/closure
        The closure of the regression method.

    Raises
    ------
    ValueError
        variance must be set to a number between 0 and 1
    """
    R, _, _, V = _svd_regression(X, T, variance=variance)
    W = np.dot(X, V.T)
    R, _, _, _ = _svd_regression(W, T)

    def mlr_pca_weights_fitting(x):
        w = np.dot(x, V.T)
        return np.dot(w, R)

    return mlr_pca_weights_fitting

def _svd_regression(X, T, variance=None):
    r"""
    SVD decomposition for regression.

    Parameters
    ----------
    X: numpy.array
        The regression features used to create the coefficient matrix.
    T: numpy.array
        The shapes differential that denote the dependent variable.
    variance: float or None, Optional
        The SVD variance.

        Default: None

    Raises
    ------
    ValueError
        variance must be set to a number between 0 and 1
    """
    if variance is not None and not (0 < variance <= 1):
        raise ValueError("variance must be set to a number between 0 and 1.")

    U, s, V = np.linalg.svd(X)
    if variance:
        total = sum(s)
        acc = 0
        for j, y in enumerate(s):
            acc += y
            if acc / total >= variance:
                r = j+1
                break
    else:
        tol = np.max(X.shape) * np.spacing(np.max(s))
        r = np.sum(s > tol)
    U = U[:, :r]
    s = 1 / s[:r]
    V = V[:r, :]
    R = np.dot(np.dot(V.T * s, U.T), T)

    return R, U, s, V
<|endoftext|>


<commit_before>import sys
import shutil
import os
from setuptools import setup, find_packages
import cobwebs
import subprocess

if len(sys.argv) > 0:
    if sys.argv[1] in ("install", "develop"):
        try:
            os.mkdir("/etc/spider/")
        except FileExistsError:
            print("Warning: /etc/spider directory already exists...")
        print("Copying file to /etc/spider/")
        ret = shutil.copy("conf/cobwebs.yaml", "/etc/spider/cobwebs.yaml")
        print(ret)
        subprocess.call(["ls", "-l", "/etc/spider"])

setup(

    name='cobwebs',

    version=cobwebs.__version__,

    packages=find_packages(),

    author="Asteroide",

    author_email="asteroide__AT__domtombox.net",

    description="A house for spider utilities",

    long_description=open('README.md').read(),

    # install_requires= ,

    include_package_data=True,

    url='https://github.com/asteroide/immo_spider',

    classifiers=[
        "Programming Language :: Python",
        "Development Status :: 1 - Planning",
        "License :: OSI Approved",
        "Natural Language :: French",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3.5",
        "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)"
    ],

)
<commit_msg>Move import of the main library.<commit_after>import sys
import shutil
import os
from setuptools import setup, find_packages
import subprocess

if len(sys.argv) > 0:
    if sys.argv[1] in ("install", "develop"):
        try:
            os.mkdir("/etc/spider/")
        except FileExistsError:
            print("Warning: /etc/spider directory already exists...")
        print("Copying file to /etc/spider/")
        ret = shutil.copy("conf/cobwebs.yaml", "/etc/spider/cobwebs.yaml")
        print(ret)
        subprocess.call(["ls", "-l", "/etc/spider"])

import cobwebs

setup(

    name='cobwebs',

    version=cobwebs.__version__,

    packages=find_packages(),

    author="Asteroide",

    author_email="asteroide__AT__domtombox.net",

    description="A house for spider utilities",

    long_description=open('README.md').read(),

    # install_requires= ,

    include_package_data=True,

    url='https://github.com/asteroide/immo_spider',

    classifiers=[
        "Programming Language :: Python",
        "Development Status :: 1 - Planning",
        "License :: OSI Approved",
        "Natural Language :: French",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3.5",
        "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)"
    ],

)
<|endoftext|>


<commit_before>import sys
import re
import logging
import html
from PyQt5.QtCore import Qt, QPoint, QRect, QSize, QEvent, QCoreApplication
from PyQt5.QtGui import QKeySequence, QValidator, QPainter, QPen, QBrush, QColor, QPixmap
from PyQt5.QtWidgets import QApplication, QTableWidget, QMenu, \
  QFrame, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QLineEdit, QLayout, \
  QComboBox, QGridLayout

import log

__all__ = ['TableWidget', 'HSeparator', 'HBoxLayout', 'VBoxLayout',
           'ErrorBaloon', 'ErrorCheckEdit', 'FlowLayout',
           'DescriptionWidget', 'ComboBoxWithDescriptor']

class TableWidget(QTableWidget):
  def __init__(self):
    super().__init__()
    self.menu = QMenu()
    self.keys = []

    self.addAction('&Copy selected', self.copySelected, QKeySequence.Copy)
    self.addAction('&Paste',         self.paste,        QKeySequence.Paste)

  def addAction(self, label, func, key):
    self.menu.addAction(label, func, key)
    self.keys.append((key, func))

  def keyPressEvent(self, ev):
    for key, func in self.keys:
      if isinstance(key, QKeySequence.StandardKey):
        m = ev.matches(key)
      else:
        m = (ev.key() | int(ev.modifiers())) in key

      if m:
        ev.accept()
        func()
        return
    super().keyPressEvent(ev)

  def contextMenuEvent(self, ev):
    self.menu.exec_(ev.globalPos())

  def copySelected(self):
    rows = []
    row = None
    curRow = None
    for cell in self.selectedIndexes():
      if cell.row() != curRow:
        row = []
        rows.append(row)
        curRow = cell.row()
      row.append(str(cell.data()).strip())
    QApplication.clipboard().setText('\n'.join(['\t'.join(r) for r in rows]))

  def paste(self):
    text = QApplication.clipboard().text()
    rows = [[c.strip() for c in l.split('\t')] for l in re.split(r'\r?\n', text)]

    r0, c0 = self.currentRow(), self.currentColumn()
    for r, row in enumerate(rows):
      for c, text in enumerate(row):
        item = self.item(r0+r, c0+c)
        if item and item.flags() & Qt.ItemIsEditable:
          item.setText(text)
          self.cellChanged.emit(item.row(), item.column())

class HSeparator(QFrame):
  def __init__(self):
    super().__init__()
    self.setFrameShape(QFrame.HLine)
    self.setFrameShadow(QFrame.Sunken)

class HBoxLayout(QHBoxLayout):
  def __init__(self):
    super().__init__()
    self.setContentsMargins(0, 0, 0, 0)
    self.setSpacing(4)

class VBoxLayout(QVBoxLayout):
  def __init__(self):
    super().__init__()
    self.setContentsMargins(0, 0, 0, 0)
    self.setSpacing(4)

class ErrorBaloon(QWidget):
  def __init__(self):
    super().__init__()

    self.label = QLabel()

    vbox = VBoxLayout()
    vbox.setContentsMargins(4, 4, 4, 4)
    vbox.addWidget(self.label)
    self.setLayout(vbox)

  def paintEvent(self, ev):
    rect = self.rect()
    rect = QRect(rect.x(), rect.y(), rect.width()-1, rect.height()-1)

    painter = QPainter(self)
    painter.setRenderHint(painter.Antialiasing)
    painter.setBrush(QBrush(QColor(0xf8, 0xf8, 0xf8)))
    painter.setPen  (  QPen(QColor(0x80, 0x80, 0x80)))
    painter.drawRoundedRect(rect, 3, 3)
    painter.end()

  def setMessage(self, text):
    self.label.setText('%s' % html.escape(text))

  def updatePosition(self, widget):
    self.setParent(widget.window())
    self.adjustSize()

    parent = widget.parentWidget()
    pt = widget.mapTo(parent, QPoint(0, 0))
    while(parent != self.parentWidget()):
      pt = parent.mapTo(parent.parentWidget(), pt)
      parent = parent.parentWidget()

    self.move(pt.x(), pt.y() - self.rect().height())

class ErrorCheckEdit(QLineEdit):
  def __init__(self, validator, *args, **kwargs):
    super().__init__(*args, **kwargs)
    self.validator = validator
    self.baloon = ErrorBaloon()
    self.state = QValidator.Acceptable
    self.textChanged.connect(lambda t: self.checkValue())

  def checkValue(self):
    try:
      self.state, message = self.validator(self.text())
    except:
      log.warnException()
      self.state = QValidator.Invalid
      message = '%s %s' % sys.exc_info()[:2]

    self.baloon.setMessage(message)
    if self.state != QValidator.Acceptable and self.hasFocus():
      self.showBaloon()
    else:
      self.hideBaloon()

  def showBaloon(self):
    self.baloon.updatePosition(self)
    self.baloon.update()
    self.baloon.show()
    self.setStyleSheet('');

  def hideBaloon(self):
    self.baloon.hide()
    if self.state == QValidator.Acceptable:
      self.setStyleSheet('');
    else:
      self.setStyleSheet('background-color:red');

  def focusInEvent(self, ev):
    super().focusInEvent(ev)
    if self.state == QValidator.Acceptable:
      self.hideBaloon()
    else:
      self.showBaloon()

  def focusOutEvent(self, ev):
    super().focusOutEvent(ev)
    self.hideBaloon()

class FlowLayout(QLayout):
  def __init__(self):
    super().__init__()
    self.items = []

  def count(self):
    return len(self.items)

  def addItem(self, item):
    self.items.append(item)

  def setGeometry(self, rect):
    super().setGeometry(rect)
    self.doLayout(rect)

  def sizeHint(self):
    if self.count() == 0:
      return QSize(0, 0)

    s = [item.minimumSize() for item in self.items]
    w = sum([i.width() for i in s])
    h = max([i.height() for i in s])
    return QSize(w, h)

  def expandingDirections(self):
    return Qt.Orientation(0)

  def hasHeightForWidth(self):
    return True

  def heightForWidth(self, width):
    return self.doLayout(QRect(0, 0, width, 0))

  def itemAt(self, idx):
    try:
      return self.items[idx]
    except IndexError:
      return None

  def takeAt(self, idx):
    try:
      item = self.items[idx]
      item.widget().close()
      del self.items[idx]
      return item
    except IndexError:
      return None

  def doLayout(self, rect):
    l, t = rect.x(), rect.y()
    r, b = l+rect.width(), t+rect.height()

    width = 0
    col, rh = 0, 0
    x, y = l, t
    for item in self.items:
      s = item.minimumSize()

      if col > 0 and x+s.width() >= r:
        x = l
        y += rh + 4
        col, rh = 0, 0

      item.setGeometry(QRect(x, y, s.width(), s.height()))
      col += 1
      rh = max([rh, s.height()])
      x += s.width()+4
      width = max(width, x-4)

    return y-t+rh

class DescriptionWidget(QFrame):
  def __init__(self):
    super().__init__()

    vbox = VBoxLayout()
    vbox.setContentsMargins(4, 4, 4, 4)
    self.setLayout(vbox)
    self.vbox = vbox

    self.setFrameShape(QFrame.StyledPanel)

  def addTitle(self, title):
    label = QLabel(title)
    label.setContentsMargins(16, 4, 16, 4)

    vbox = VBoxLayout()
    vbox.addWidget(label)

    frame = QFrame()
    frame.setFrameShape(QFrame.StyledPanel)
    frame.setContentsMargins(4, 4, 4, 4)
    frame.setLayout(vbox)
    self.vbox.addWidget(frame)

  def addLabel(self, text, **kwargs):
    label = QLabel(text)
    if kwargs.get('richtext'):
      label.setTextFormat(Qt.RichText)
      label.setTextInteractionFlags(Qt.TextBrowserInteraction)
      label.setOpenExternalLinks(True)
    self.vbox.addWidget(label)

  def addImage(self, image):
    imglabel = QLabel()
    imglabel.setContentsMargins(16, 4, 16, 4)
    imglabel.setPixmap(QPixmap.fromImage(image))
    self.vbox.addWidget(imglabel)

  def addGrid(self):
    grid = QGridLayout()
    grid.setContentsMargins(16, 4, 4, 16)
    grid.setColumnStretch(1, 1)
    grid.setHorizontalSpacing(16)
    self.vbox.addLayout(grid)
    return grid

class ComboBoxWithDescriptor(QComboBox):
  def __init__(self):
    super().__init__()

    self.currDescriptor = None
    self.preventHide = False

    self.view().entered.connect(self.showDescriptor)
    self.view().window().installEventFilter(self)

  def closeDescriptor(self):
    if self.currDescriptor:
      self.currDescriptor.close()
      self.currDescriptor = None

  def showDescriptor(self, index):
    self.closeDescriptor()

    widget = index.data(Qt.UserRole+1)
    if not isinstance(widget, QWidget):
      return

    widget.setWindowFlags(Qt.ToolTip)

    view = self.view()
    pos = view.mapToGlobal(QPoint(view.width(), view.visualRect(index).y()))

    widget.move(pos)
    widget.show()

    self.currDescriptor = widget

  def eventFilter(self, obj, event):
    if obj == self.view().window():
      if event.type() in (QEvent.MouseButtonPress, QEvent.MouseButtonRelease):
        d = self.currDescriptor
        if d and d.geometry().contains(event.globalPos()):
          self.preventHide = True

    return False

  def hidePopup(self):
    if self.preventHide:
      self.preventHide = False
      return

    super().hidePopup()
    self.closeDescriptor()
<commit_msg>Fix copy and paste in TableWidgets that have different visual and logical (columns|rows)<commit_after>import sys
import re
import logging
import html
from PyQt5.QtCore import Qt, QPoint, QRect, QSize, QEvent, QCoreApplication
from PyQt5.QtGui import QKeySequence, QValidator, QPainter, QPen, QBrush, QColor, QPixmap
from PyQt5.QtWidgets import QApplication, QTableWidget, QMenu, \
  QFrame, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QLineEdit, QLayout, \
  QComboBox, QGridLayout
import numpy as np

import log

__all__ = ['TableWidget', 'HSeparator', 'HBoxLayout', 'VBoxLayout',
           'ErrorBaloon', 'ErrorCheckEdit', 'FlowLayout',
           'DescriptionWidget', 'ComboBoxWithDescriptor']

class TableWidget(QTableWidget):
  def __init__(self):
    super().__init__()
    self.menu = QMenu()
    self.keys = []

    self.addAction('&Copy selected', self.copySelected, QKeySequence.Copy)
    self.addAction('&Paste',         self.paste,        QKeySequence.Paste)

  def addAction(self, label, func, key):
    self.menu.addAction(label, func, key)
    self.keys.append((key, func))

  def keyPressEvent(self, ev):
    for key, func in self.keys:
      if isinstance(key, QKeySequence.StandardKey):
        m = ev.matches(key)
      else:
        m = (ev.key() | int(ev.modifiers())) in key

      if m:
        ev.accept()
        func()
        return
    super().keyPressEvent(ev)

  def contextMenuEvent(self, ev):
    self.menu.exec_(ev.globalPos())

  def copySelected(self):
    # rows = []
    # row = None
    # curRow = None

    col, row, val = [], [], []
    for index in self.selectedIndexes():
      col.append(self.visualColumn(index.column()))
      row.append(self.visualRow(index.row()))
      val.append(str(index.data()).strip())

    col = np.array(col) - min(col)
    row = np.array(row) - min(row)

    data = dict(zip(zip(row, col), val))
    tbl = [[data[(r, c)] for c in range(max(col)+1)] for r in range(max(row)+1)]

    QApplication.clipboard().setText('\n'.join(['\t'.join(r) for r in tbl]))

  def paste(self):
    text = QApplication.clipboard().text()
    rows = [[c.strip() for c in l.split('\t')] for l in re.split(r'\r?\n', text)]

    v2l_r = dict([(self.visualRow(r), r)    for r in range(self.rowCount())])
    v2l_c = dict([(self.visualColumn(c), c) for c in range(self.columnCount())])

    r0 = self.visualRow(self.currentRow())
    c0 = self.visualColumn(self.currentColumn())
    for r, vals in enumerate(rows):
      for c, text in enumerate(vals):
        item = self.item(v2l_r[r0+r], v2l_c[c0+c])
        if item and item.flags() & Qt.ItemIsEditable:
          item.setText(text)
          self.cellChanged.emit(item.row(), item.column())

class HSeparator(QFrame):
  def __init__(self):
    super().__init__()
    self.setFrameShape(QFrame.HLine)
    self.setFrameShadow(QFrame.Sunken)

class HBoxLayout(QHBoxLayout):
  def __init__(self):
    super().__init__()
    self.setContentsMargins(0, 0, 0, 0)
    self.setSpacing(4)

class VBoxLayout(QVBoxLayout):
  def __init__(self):
    super().__init__()
    self.setContentsMargins(0, 0, 0, 0)
    self.setSpacing(4)

class ErrorBaloon(QWidget):
  def __init__(self):
    super().__init__()

    self.label = QLabel()

    vbox = VBoxLayout()
    vbox.setContentsMargins(4, 4, 4, 4)
    vbox.addWidget(self.label)
    self.setLayout(vbox)

  def paintEvent(self, ev):
    rect = self.rect()
    rect = QRect(rect.x(), rect.y(), rect.width()-1, rect.height()-1)

    painter = QPainter(self)
    painter.setRenderHint(painter.Antialiasing)
    painter.setBrush(QBrush(QColor(0xf8, 0xf8, 0xf8)))
    painter.setPen  (  QPen(QColor(0x80, 0x80, 0x80)))
    painter.drawRoundedRect(rect, 3, 3)
    painter.end()

  def setMessage(self, text):
    self.label.setText('%s' % html.escape(text))

  def updatePosition(self, widget):
    self.setParent(widget.window())
    self.adjustSize()

    parent = widget.parentWidget()
    pt = widget.mapTo(parent, QPoint(0, 0))
    while(parent != self.parentWidget()):
      pt = parent.mapTo(parent.parentWidget(), pt)
      parent = parent.parentWidget()

    self.move(pt.x(), pt.y() - self.rect().height())

class ErrorCheckEdit(QLineEdit):
  def __init__(self, validator, *args, **kwargs):
    super().__init__(*args, **kwargs)
    self.validator = validator
    self.baloon = ErrorBaloon()
    self.state = QValidator.Acceptable
    self.textChanged.connect(lambda t: self.checkValue())

  def checkValue(self):
    try:
      self.state, message = self.validator(self.text())
    except:
      log.warnException()
      self.state = QValidator.Invalid
      message = '%s %s' % sys.exc_info()[:2]

    self.baloon.setMessage(message)
    if self.state != QValidator.Acceptable and self.hasFocus():
      self.showBaloon()
    else:
      self.hideBaloon()

  def showBaloon(self):
    self.baloon.updatePosition(self)
    self.baloon.update()
    self.baloon.show()
    self.setStyleSheet('');

  def hideBaloon(self):
    self.baloon.hide()
    if self.state == QValidator.Acceptable:
      self.setStyleSheet('');
    else:
      self.setStyleSheet('background-color:red');

  def focusInEvent(self, ev):
    super().focusInEvent(ev)
    if self.state == QValidator.Acceptable:
      self.hideBaloon()
    else:
      self.showBaloon()

  def focusOutEvent(self, ev):
    super().focusOutEvent(ev)
    self.hideBaloon()

class FlowLayout(QLayout):
  def __init__(self):
    super().__init__()
    self.items = []

  def count(self):
    return len(self.items)

  def addItem(self, item):
    self.items.append(item)

  def setGeometry(self, rect):
    super().setGeometry(rect)
    self.doLayout(rect)

  def sizeHint(self):
    if self.count() == 0:
      return QSize(0, 0)

    s = [item.minimumSize() for item in self.items]
    w = sum([i.width() for i in s])
    h = max([i.height() for i in s])
    return QSize(w, h)

  def expandingDirections(self):
    return Qt.Orientation(0)

  def hasHeightForWidth(self):
    return True

  def heightForWidth(self, width):
    return self.doLayout(QRect(0, 0, width, 0))

  def itemAt(self, idx):
    try:
      return self.items[idx]
    except IndexError:
      return None

  def takeAt(self, idx):
    try:
      item = self.items[idx]
      item.widget().close()
      del self.items[idx]
      return item
    except IndexError:
      return None

  def doLayout(self, rect):
    l, t = rect.x(), rect.y()
    r, b = l+rect.width(), t+rect.height()

    width = 0
    col, rh = 0, 0
    x, y = l, t
    for item in self.items:
      s = item.minimumSize()

      if col > 0 and x+s.width() >= r:
        x = l
        y += rh + 4
        col, rh = 0, 0

      item.setGeometry(QRect(x, y, s.width(), s.height()))
      col += 1
      rh = max([rh, s.height()])
      x += s.width()+4
      width = max(width, x-4)

    return y-t+rh

class DescriptionWidget(QFrame):
  def __init__(self):
    super().__init__()

    vbox = VBoxLayout()
    vbox.setContentsMargins(4, 4, 4, 4)
    self.setLayout(vbox)
    self.vbox = vbox

    self.setFrameShape(QFrame.StyledPanel)

  def addTitle(self, title):
    label = QLabel(title)
    label.setContentsMargins(16, 4, 16, 4)

    vbox = VBoxLayout()
    vbox.addWidget(label)

    frame = QFrame()
    frame.setFrameShape(QFrame.StyledPanel)
    frame.setContentsMargins(4, 4, 4, 4)
    frame.setLayout(vbox)
    self.vbox.addWidget(frame)

  def addLabel(self, text, **kwargs):
    label = QLabel(text)
    if kwargs.get('richtext'):
      label.setTextFormat(Qt.RichText)
      label.setTextInteractionFlags(Qt.TextBrowserInteraction)
      label.setOpenExternalLinks(True)
    self.vbox.addWidget(label)

  def addImage(self, image):
    imglabel = QLabel()
    imglabel.setContentsMargins(16, 4, 16, 4)
    imglabel.setPixmap(QPixmap.fromImage(image))
    self.vbox.addWidget(imglabel)

  def addGrid(self):
    grid = QGridLayout()
    grid.setContentsMargins(16, 4, 4, 16)
    grid.setColumnStretch(1, 1)
    grid.setHorizontalSpacing(16)
    self.vbox.addLayout(grid)
    return grid

class ComboBoxWithDescriptor(QComboBox):
  def __init__(self):
    super().__init__()

    self.currDescriptor = None
    self.preventHide = False

    self.view().entered.connect(self.showDescriptor)
    self.view().window().installEventFilter(self)

  def closeDescriptor(self):
    if self.currDescriptor:
      self.currDescriptor.close()
      self.currDescriptor = None

  def showDescriptor(self, index):
    self.closeDescriptor()

    widget = index.data(Qt.UserRole+1)
    if not isinstance(widget, QWidget):
      return

    widget.setWindowFlags(Qt.ToolTip)

    view = self.view()
    pos = view.mapToGlobal(QPoint(view.width(), view.visualRect(index).y()))

    widget.move(pos)
    widget.show()

    self.currDescriptor = widget

  def eventFilter(self, obj, event):
    if obj == self.view().window():
      if event.type() in (QEvent.MouseButtonPress, QEvent.MouseButtonRelease):
        d = self.currDescriptor
        if d and d.geometry().contains(event.globalPos()):
          self.preventHide = True

    return False

  def hidePopup(self):
    if self.preventHide:
      self.preventHide = False
      return

    super().hidePopup()
    self.closeDescriptor()
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.contrib import messages
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views.generic.edit import ProcessFormView
from django.views.generic.base import TemplateResponseMixin, ContextMixin, TemplateView, View
from braces.views import LoginRequiredMixin, CsrfExemptMixin
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponseRedirect
from django.conf import settings
# Create your views here.
from accounts.models import User
from .models import *
from django.http import HttpResponse, HttpResponseNotFound
from django.core.exceptions import ObjectDoesNotExist
from django.http import Http404

class CourseListView(LoginRequiredMixin, ListView):
    messages = {
        'invalid_enroll_id': 'Unable to find the course you want to enroll to. Please contact admins',
        'course_closed': 'Enrollments for this course are closed at this time.'
    }
    model = Course
    context_object_name = 'courses'
    template_name = "course_list.html"
    success_url = reverse_lazy('courses:course-list')

    show_other = False
    show_closed = False

    # custom queryset called by the listview, if getEnroll = False avoid to get enroll_id from request (used by post request).
    # Dont change the default due to the listview compatibility
    def get_queryset(self):
        self.show_other = self.request.GET.get('show_other') == 'True'
        self.show_closed = self.request.GET.get('show_closed') == 'True'

        if not self.show_closed:
            dataSet = self.model.objects.filter_ongoing()
        else:
            dataSet = self.model.objects.all()
        if not self.show_other:
            dataSet = dataSet.filter(Students=self.request.user)
        else:
            dataSet = dataSet.exclude(Students=self.request.user)
        return dataSet

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(CourseListView, self).get_context_data(**kwargs)

        context['mycourses'] = self._get_mycourses(context['object_list'])
        context['show_other'] = self.show_other
        context['show_closed'] = self.show_closed
        return context

    def post(self, request):
        # enroll_id corresponds to the course id
        enroll_id = self.request.POST.get('enroll_id') or False
        enroll_id = self._clean_enroll_id(enroll_id)
        if not enroll_id:
            redirect_url = self.request.get_full_path()
            return HttpResponseRedirect(redirect_url)
        else:
            course = self.model.objects.get(id=enroll_id)
            course.enroll(self.request.user.id)
            return HttpResponseRedirect(self.success_url)

    def _clean_enroll_id(self, enroll_id):
        """
        Verify that a course with this ID exists and is open for enrollments
        """
        try:
            enroll_id = int(enroll_id)
            course = self.model.objects.get(id=enroll_id)
        except:
            course = False
        if not course:
            messages.error(self.request, self.messages['invalid_enroll_id'])
            return False
        if not course.Is_enrollment_open():
            messages.error(self.request, self.messages['course_closed'])
            return False
        return enroll_id

    # list of courses to which the user is enrolled
    def _get_mycourses(self, courses_list):
        mycourses = []
        for course in courses_list.filter(Students=self.request.user):
            mycourses.append(course.id)
        return mycourses

class AssignmentListView(LoginRequiredMixin, ListView):

    template_name = "assignment_list.html"
    context_object_name = 'courses'

    def get_queryset(self):
        courses_list = Course.objects.filter(Students=self.request.user)
        return courses_list

    def _get_result_list(self, ex_list):
        result_list = []
        for e in ex_list:
            res_set = e.result_set.filter(Pass=True).order_by('-Creation_date')
            if not res_set:
                res_set = e.result_set.filter(Pass=False).order_by('-Creation_date')
            if res_set:
                result_list.append(res_set[0])
        return result_list

    def get_context_data(self, **kwargs):
        context = super(AssignmentListView, self).get_context_data(**kwargs)
        courses_list = context['courses']
        if courses_list:
            selected_course_id = self.request.GET.get('selected_course_id') or False
            if selected_course_id:
                try:
                    selected_course = courses_list.get(id=selected_course_id)
                except:
                    selected_course = courses_list[0]
            else:
                selected_course = courses_list[0]
            assignment_list = Assignment.objects.filter(Course=selected_course)
            exercise_list = self.request.user.exercise_set.all()
            result_list = self._get_result_list(exercise_list)
            context['assignments'] = assignment_list
            context['exercises'] = exercise_list
            print 'xxx', result_list
            context['results'] = result_list
            context['selected_course'] = selected_course

        return context

class DownloadUserFileView(LoginRequiredMixin, View):
    def get(self, request, *args, **kwargs):
        try:
            ex_id = request.GET.get('ex_id') or False
            e = Exercise.objects.get(id=ex_id)

            #filename = 'ex_package.tar.gz'
            #filepath = join(settings.USER_DATA_ROOT, str(e.Folder_path), filename)
            package = UserFile.objects.get(Exercise=e, Type='package')
            filename = package.Name
            filepath = join(settings.USER_DATA_ROOT, package.Folder_path, filename)

            # You got the zip! Now, return it!
            response = HttpResponse(open(filepath, 'rb'), content_type='application/x-gtar')
            response['Content-Disposition'] = 'attachment; filename=%s' % filename
            return response
        except:
            #return HttpResponseNotFound(filepath)
            return HttpResponseNotFound('Exercise package not found. Please contact the admins.')

class ResultListView(ListView):
    model = Result
    template_name = "result_list.html"
    context_object_name = 'results'<commit_msg>now the students see only their own exercises in the history<commit_after>from django.shortcuts import render
from django.contrib import messages
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views.generic.edit import ProcessFormView
from django.views.generic.base import TemplateResponseMixin, ContextMixin, TemplateView, View
from braces.views import LoginRequiredMixin, CsrfExemptMixin
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponseRedirect
from django.conf import settings
# Create your views here.
from accounts.models import User
from .models import *
from django.http import HttpResponse, HttpResponseNotFound
from django.core.exceptions import ObjectDoesNotExist
from django.http import Http404

class CourseListView(LoginRequiredMixin, ListView):
    messages = {
        'invalid_enroll_id': 'Unable to find the course you want to enroll to. Please contact admins',
        'course_closed': 'Enrollments for this course are closed at this time.'
    }
    model = Course
    context_object_name = 'courses'
    template_name = "course_list.html"
    success_url = reverse_lazy('courses:course-list')

    show_other = False
    show_closed = False

    # custom queryset called by the listview, if getEnroll = False avoid to get enroll_id from request (used by post request).
    # Dont change the default due to the listview compatibility
    def get_queryset(self):
        self.show_other = self.request.GET.get('show_other') == 'True'
        self.show_closed = self.request.GET.get('show_closed') == 'True'

        if not self.show_closed:
            dataSet = self.model.objects.filter_ongoing()
        else:
            dataSet = self.model.objects.all()
        if not self.show_other:
            dataSet = dataSet.filter(Students=self.request.user)
        else:
            dataSet = dataSet.exclude(Students=self.request.user)
        return dataSet

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(CourseListView, self).get_context_data(**kwargs)

        context['mycourses'] = self._get_mycourses(context['object_list'])
        context['show_other'] = self.show_other
        context['show_closed'] = self.show_closed
        return context

    def post(self, request):
        # enroll_id corresponds to the course id
        enroll_id = self.request.POST.get('enroll_id') or False
        enroll_id = self._clean_enroll_id(enroll_id)
        if not enroll_id:
            redirect_url = self.request.get_full_path()
            return HttpResponseRedirect(redirect_url)
        else:
            course = self.model.objects.get(id=enroll_id)
            course.enroll(self.request.user.id)
            return HttpResponseRedirect(self.success_url)

    def _clean_enroll_id(self, enroll_id):
        """
        Verify that a course with this ID exists and is open for enrollments
        """
        try:
            enroll_id = int(enroll_id)
            course = self.model.objects.get(id=enroll_id)
        except:
            course = False
        if not course:
            messages.error(self.request, self.messages['invalid_enroll_id'])
            return False
        if not course.Is_enrollment_open():
            messages.error(self.request, self.messages['course_closed'])
            return False
        return enroll_id

    # list of courses to which the user is enrolled
    def _get_mycourses(self, courses_list):
        mycourses = []
        for course in courses_list.filter(Students=self.request.user):
            mycourses.append(course.id)
        return mycourses

class AssignmentListView(LoginRequiredMixin, ListView):

    template_name = "assignment_list.html"
    context_object_name = 'courses'

    def get_queryset(self):
        courses_list = Course.objects.filter(Students=self.request.user)
        return courses_list

    def _get_result_list(self, ex_list):
        result_list = []
        for e in ex_list:
            res_set = e.result_set.filter(Pass=True).order_by('-Creation_date')
            if not res_set:
                res_set = e.result_set.filter(Pass=False).order_by('-Creation_date')
            if res_set:
                result_list.append(res_set[0])
        return result_list

    def get_context_data(self, **kwargs):
        context = super(AssignmentListView, self).get_context_data(**kwargs)
        courses_list = context['courses']
        if courses_list:
            selected_course_id = self.request.GET.get('selected_course_id') or False
            if selected_course_id:
                try:
                    selected_course = courses_list.get(id=selected_course_id)
                except:
                    selected_course = courses_list[0]
            else:
                selected_course = courses_list[0]
            assignment_list = Assignment.objects.filter(Course=selected_course)
            exercise_list = self.request.user.exercise_set.all()
            result_list = self._get_result_list(exercise_list)
            context['assignments'] = assignment_list
            context['exercises'] = exercise_list
            print 'xxx', result_list
            context['results'] = result_list
            context['selected_course'] = selected_course

        return context

class DownloadUserFileView(LoginRequiredMixin, View):
    def get(self, request, *args, **kwargs):
        try:
            ex_id = request.GET.get('ex_id') or False
            e = Exercise.objects.get(id=ex_id)

            #filename = 'ex_package.tar.gz'
            #filepath = join(settings.USER_DATA_ROOT, str(e.Folder_path), filename)
            package = UserFile.objects.get(Exercise=e, Type='package')
            filename = package.Name
            filepath = join(settings.USER_DATA_ROOT, package.Folder_path, filename)

            # You got the zip! Now, return it!
            response = HttpResponse(open(filepath, 'rb'), content_type='application/x-gtar')
            response['Content-Disposition'] = 'attachment; filename=%s' % filename
            return response
        except:
            #return HttpResponseNotFound(filepath)
            return HttpResponseNotFound('Exercise package not found. Please contact the admins.')

class ResultListView(LoginRequiredMixin, ListView):
    template_name = "result_list.html"
    context_object_name = 'results'

    def get_queryset(self):
        courses_list = Result.objects.order_by('-Creation_date').filter(User=self.request.user)
        return courses_list
<|endoftext|>


<commit_before>"""
desispec.sky
============

Utility functions to compute a sky model and subtract it.
"""

import numpy as np
from desispec.resolution import Resolution
from desispec.linalg import cholesky_solve
from desispec.linalg import cholesky_solve_and_invert
from desispec.linalg import spline_fit
from desispec.log import get_logger
from desispec import util
from desiutil import stats as dustat
import scipy,scipy.sparse,scipy.stats,scipy.ndimage
import sys

def compute_sky(frame, nsig_clipping=4.,max_iterations=100) :
    """Compute a sky model.

    Input has to correspond to sky fibers only.
    Input flux are expected to be flatfielded!
    We don't check this in this routine.

    Args:
        frame : Frame object, which includes attributes
          - wave : 1D wavelength grid in Angstroms
          - flux : 2D flux[nspec, nwave] density
          - ivar : 2D inverse variance of flux
          - mask : 2D inverse mask flux (0=good)
          - resolution_data : 3D[nspec, ndiag, nwave]  (only sky fibers)
        nsig_clipping : [optional] sigma clipping value for outlier rejection

    returns SkyModel object with attributes wave, flux, ivar, mask
    """

    log=get_logger()
    log.info("starting")

    # Grab sky fibers on this frame
    skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
    assert np.max(skyfibers) < 500  #- indices, not fiber numbers

    nwave=frame.nwave
    nfibers=len(skyfibers)

    current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0)
    flux = frame.flux[skyfibers]
    Rsky = frame.R[skyfibers]

    sqrtw=np.sqrt(current_ivar)
    sqrtwflux=sqrtw*flux

    chi2=np.zeros(flux.shape)

    #debug
    #nfibers=min(nfibers,2)

    nout_tot=0
    for iteration in range(max_iterations) :

        A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
        B=np.zeros((nwave))
        # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
        SD=scipy.sparse.lil_matrix((nwave,nwave))
        # loop on fiber to handle resolution
        for fiber in range(nfibers) :
            if fiber%10==0 :
                log.info("iter %d fiber %d"%(iteration,fiber))
            R = Rsky[fiber]

            # diagonal sparse matrix with content = sqrt(ivar)
            SD.setdiag(sqrtw[fiber])

            sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r]

            A = A+(sqrtwR.T*sqrtwR).tocsr()
            B += sqrtwR.T*sqrtwflux[fiber]

        log.info("iter %d solving"%iteration)

        w = A.diagonal()>0
        A_pos_def = A.todense()[w,:]
        A_pos_def = A_pos_def[:,w]
        skyflux = B*0
        try:
            skyflux[w]=cholesky_solve(A_pos_def,B[w])
        except:
            log.info("cholesky failed, trying svd in iteration {}".format(iteration))
            skyflux[w]=np.linalg.lstsq(A_pos_def,B[w])[0]

        log.info("iter %d compute chi2"%iteration)

        for fiber in range(nfibers) :

            S = Rsky[fiber].dot(skyflux)
            chi2[fiber]=current_ivar[fiber]*(flux[fiber]-S)**2

        log.info("rejecting")

        nout_iter=0
        if iteration<1 :
            # only remove worst outlier per wave
            # apply rejection iteratively, only one entry per wave among fibers
            # find waves with outlier (fastest way)
            nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
            selection=np.where(nout_per_wave>0)[0]
            for i in selection :
                worst_entry=np.argmax(chi2[:,i])
                current_ivar[worst_entry,i]=0
                sqrtw[worst_entry,i]=0
                sqrtwflux[worst_entry,i]=0
                nout_iter += 1

        else :
            # remove all of them at once
            bad=(chi2>nsig_clipping**2)
            current_ivar *= (bad==0)
            sqrtw *= (bad==0)
            sqrtwflux *= (bad==0)
            nout_iter += np.sum(bad)

        nout_tot += nout_iter

        sum_chi2=float(np.sum(chi2))
        ndf=int(np.sum(chi2>0)-nwave)
        chi2pdf=0.
        if ndf>0 :
            chi2pdf=sum_chi2/ndf
        log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))

        if nout_iter == 0 :
            break

    log.info("nout tot=%d"%nout_tot)

    # solve once again to get deconvolved sky variance
    try :
        skyflux,skycovar=cholesky_solve_and_invert(A.todense(),B)
    except np.linalg.linalg.LinAlgError,e :
        log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv")
        skyflux = np.linalg.lstsq(A.todense(),B)[0]
        skycovar = np.linalg.pinv(A.todense())

    #- sky inverse variance, but incomplete and not needed anyway
    # skyvar=np.diagonal(skycovar)
    # skyivar=(skyvar>0)/(skyvar+(skyvar==0))

    # Use diagonal of skycovar convolved with mean resolution of all fibers
    # first compute average resolution
    mean_res_data=np.mean(frame.resolution_data,axis=0)
    R = Resolution(mean_res_data)
    # compute convolved sky and ivar
    cskycovar=R.dot(skycovar).dot(R.T.todense())
    cskyvar=np.diagonal(cskycovar)
    cskyivar=(cskyvar>0)/(cskyvar+(cskyvar==0))

    # convert cskyivar to 2D; today it is the same for all spectra,
    # but that may not be the case in the future
    cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave)

    # Convolved sky
    cskyflux = np.zeros(frame.flux.shape)
    for i in range(frame.nspec):
        cskyflux[i] = frame.R[i].dot(skyflux)

    # need to do better here
    mask = (cskyivar==0).astype(np.uint32)

    return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask,
                    nrej=nout_tot)

class SkyModel(object):
    def __init__(self, wave, flux, ivar, mask, header=None, nrej=0):
        """Create SkyModel object

        Args:
            wave  : 1D[nwave] wavelength in Angstroms
            flux  : 2D[nspec, nwave] sky model to subtract
            ivar  : 2D[nspec, nwave] inverse variance of the sky model
            mask  : 2D[nspec, nwave] 0=ok or >0 if problems; 32-bit
            header : (optional) header from FITS file HDU0
            nrej : (optional) Number of rejected pixels in fit

        All input arguments become attributes
        """
        assert wave.ndim == 1
        assert flux.ndim == 2
        assert ivar.shape == flux.shape
        assert mask.shape == flux.shape

        self.nspec, self.nwave = flux.shape
        self.wave = wave
        self.flux = flux
        self.ivar = ivar
        self.mask = util.mask32(mask)
        self.header = header
        self.nrej = nrej

def subtract_sky(frame, skymodel) :
    """Subtract skymodel from frame, altering frame.flux, .ivar, and .mask

    Args:
        frame : desispec.Frame object
        skymodel : desispec.SkyModel object
    """
    assert frame.nspec == skymodel.nspec
    assert frame.nwave == skymodel.nwave

    log=get_logger()
    log.info("starting")

    # check same wavelength, die if not the case
    if not np.allclose(frame.wave, skymodel.wave):
        message = "frame and sky not on same wavelength grid"
        log.error(message)
        raise ValueError(message)

    frame.flux -= skymodel.flux
    frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
    frame.mask |= skymodel.mask

    log.info("done")

def qa_skysub(param, frame, skymodel, quick_look=False):
    """Calculate QA on SkySubtraction

    Note: Pixels rejected in generating the SkyModel (as above), are
    not rejected in the stats calculated here.  Would need to carry
    along current_ivar to do so.

    Args:
        param : dict of QA parameters : see qa_frame.init_skysub for example
        frame : desispec.Frame object
        skymodel : desispec.SkyModel object
        quick_look : bool, optional
          If True, do QuickLook specific QA (or avoid some)
    Returns:
        qadict: dict of QA outputs
          Need to record simple Python objects for yaml (str, float, int)
    """
    from desispec.qa import qalib
    import copy

    #- QAs
    #- first subtract sky to get the sky subtracted frame. This is only for QA. Pipeline does it separately.
    tempframe=copy.deepcopy(frame) #- make a copy so as to propagate frame unaffected so that downstream pipeline uses it.
    subtract_sky(tempframe,skymodel) #- Note: sky subtract is done to get residuals. As part of pipeline it is done in fluxcalib stage

    qadict = qalib.sky_resid(param, tempframe, skymodel, quick_look=quick_look)

    qadict_snr = qalib.SignalVsNoise(tempframe,param)
    qadict.update(qadict_snr)

    return qadict

<commit_msg>fix difference of exception handling in python 2.7 and python 3.5<commit_after>"""
desispec.sky
============

Utility functions to compute a sky model and subtract it.
"""

import numpy as np
from desispec.resolution import Resolution
from desispec.linalg import cholesky_solve
from desispec.linalg import cholesky_solve_and_invert
from desispec.linalg import spline_fit
from desispec.log import get_logger
from desispec import util
from desiutil import stats as dustat
import scipy,scipy.sparse,scipy.stats,scipy.ndimage
import sys

def compute_sky(frame, nsig_clipping=4.,max_iterations=100) :
    """Compute a sky model.

    Input has to correspond to sky fibers only.
    Input flux are expected to be flatfielded!
    We don't check this in this routine.

    Args:
        frame : Frame object, which includes attributes
          - wave : 1D wavelength grid in Angstroms
          - flux : 2D flux[nspec, nwave] density
          - ivar : 2D inverse variance of flux
          - mask : 2D inverse mask flux (0=good)
          - resolution_data : 3D[nspec, ndiag, nwave]  (only sky fibers)
        nsig_clipping : [optional] sigma clipping value for outlier rejection

    returns SkyModel object with attributes wave, flux, ivar, mask
    """

    log=get_logger()
    log.info("starting")

    # Grab sky fibers on this frame
    skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
    assert np.max(skyfibers) < 500  #- indices, not fiber numbers

    nwave=frame.nwave
    nfibers=len(skyfibers)

    current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0)
    flux = frame.flux[skyfibers]
    Rsky = frame.R[skyfibers]

    sqrtw=np.sqrt(current_ivar)
    sqrtwflux=sqrtw*flux

    chi2=np.zeros(flux.shape)

    #debug
    #nfibers=min(nfibers,2)

    nout_tot=0
    for iteration in range(max_iterations) :

        A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
        B=np.zeros((nwave))
        # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
        SD=scipy.sparse.lil_matrix((nwave,nwave))
        # loop on fiber to handle resolution
        for fiber in range(nfibers) :
            if fiber%10==0 :
                log.info("iter %d fiber %d"%(iteration,fiber))
            R = Rsky[fiber]

            # diagonal sparse matrix with content = sqrt(ivar)
            SD.setdiag(sqrtw[fiber])

            sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r]

            A = A+(sqrtwR.T*sqrtwR).tocsr()
            B += sqrtwR.T*sqrtwflux[fiber]

        log.info("iter %d solving"%iteration)

        w = A.diagonal()>0
        A_pos_def = A.todense()[w,:]
        A_pos_def = A_pos_def[:,w]
        skyflux = B*0
        try:
            skyflux[w]=cholesky_solve(A_pos_def,B[w])
        except:
            log.info("cholesky failed, trying svd in iteration {}".format(iteration))
            skyflux[w]=np.linalg.lstsq(A_pos_def,B[w])[0]

        log.info("iter %d compute chi2"%iteration)

        for fiber in range(nfibers) :

            S = Rsky[fiber].dot(skyflux)
            chi2[fiber]=current_ivar[fiber]*(flux[fiber]-S)**2

        log.info("rejecting")

        nout_iter=0
        if iteration<1 :
            # only remove worst outlier per wave
            # apply rejection iteratively, only one entry per wave among fibers
            # find waves with outlier (fastest way)
            nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
            selection=np.where(nout_per_wave>0)[0]
            for i in selection :
                worst_entry=np.argmax(chi2[:,i])
                current_ivar[worst_entry,i]=0
                sqrtw[worst_entry,i]=0
                sqrtwflux[worst_entry,i]=0
                nout_iter += 1

        else :
            # remove all of them at once
            bad=(chi2>nsig_clipping**2)
            current_ivar *= (bad==0)
            sqrtw *= (bad==0)
            sqrtwflux *= (bad==0)
            nout_iter += np.sum(bad)

        nout_tot += nout_iter

        sum_chi2=float(np.sum(chi2))
        ndf=int(np.sum(chi2>0)-nwave)
        chi2pdf=0.
        if ndf>0 :
            chi2pdf=sum_chi2/ndf
        log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))

        if nout_iter == 0 :
            break

    log.info("nout tot=%d"%nout_tot)

    # solve once again to get deconvolved sky variance
    try :
        skyflux,skycovar=cholesky_solve_and_invert(A.todense(),B)
    except np.linalg.linalg.LinAlgError :
        log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv")
        skyflux = np.linalg.lstsq(A.todense(),B)[0]
        skycovar = np.linalg.pinv(A.todense())

    #- sky inverse variance, but incomplete and not needed anyway
    # skyvar=np.diagonal(skycovar)
    # skyivar=(skyvar>0)/(skyvar+(skyvar==0))

    # Use diagonal of skycovar convolved with mean resolution of all fibers
    # first compute average resolution
    mean_res_data=np.mean(frame.resolution_data,axis=0)
    R = Resolution(mean_res_data)
    # compute convolved sky and ivar
    cskycovar=R.dot(skycovar).dot(R.T.todense())
    cskyvar=np.diagonal(cskycovar)
    cskyivar=(cskyvar>0)/(cskyvar+(cskyvar==0))

    # convert cskyivar to 2D; today it is the same for all spectra,
    # but that may not be the case in the future
    cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave)

    # Convolved sky
    cskyflux = np.zeros(frame.flux.shape)
    for i in range(frame.nspec):
        cskyflux[i] = frame.R[i].dot(skyflux)

    # need to do better here
    mask = (cskyivar==0).astype(np.uint32)

    return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask,
                    nrej=nout_tot)

class SkyModel(object):
    def __init__(self, wave, flux, ivar, mask, header=None, nrej=0):
        """Create SkyModel object

        Args:
            wave  : 1D[nwave] wavelength in Angstroms
            flux  : 2D[nspec, nwave] sky model to subtract
            ivar  : 2D[nspec, nwave] inverse variance of the sky model
            mask  : 2D[nspec, nwave] 0=ok or >0 if problems; 32-bit
            header : (optional) header from FITS file HDU0
            nrej : (optional) Number of rejected pixels in fit

        All input arguments become attributes
        """
        assert wave.ndim == 1
        assert flux.ndim == 2
        assert ivar.shape == flux.shape
        assert mask.shape == flux.shape

        self.nspec, self.nwave = flux.shape
        self.wave = wave
        self.flux = flux
        self.ivar = ivar
        self.mask = util.mask32(mask)
        self.header = header
        self.nrej = nrej

def subtract_sky(frame, skymodel) :
    """Subtract skymodel from frame, altering frame.flux, .ivar, and .mask

    Args:
        frame : desispec.Frame object
        skymodel : desispec.SkyModel object
    """
    assert frame.nspec == skymodel.nspec
    assert frame.nwave == skymodel.nwave

    log=get_logger()
    log.info("starting")

    # check same wavelength, die if not the case
    if not np.allclose(frame.wave, skymodel.wave):
        message = "frame and sky not on same wavelength grid"
        log.error(message)
        raise ValueError(message)

    frame.flux -= skymodel.flux
    frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
    frame.mask |= skymodel.mask

    log.info("done")

def qa_skysub(param, frame, skymodel, quick_look=False):
    """Calculate QA on SkySubtraction

    Note: Pixels rejected in generating the SkyModel (as above), are
    not rejected in the stats calculated here.  Would need to carry
    along current_ivar to do so.

    Args:
        param : dict of QA parameters : see qa_frame.init_skysub for example
        frame : desispec.Frame object
        skymodel : desispec.SkyModel object
        quick_look : bool, optional
          If True, do QuickLook specific QA (or avoid some)
    Returns:
        qadict: dict of QA outputs
          Need to record simple Python objects for yaml (str, float, int)
    """
    from desispec.qa import qalib
    import copy

    #- QAs
    #- first subtract sky to get the sky subtracted frame. This is only for QA. Pipeline does it separately.
    tempframe=copy.deepcopy(frame) #- make a copy so as to propagate frame unaffected so that downstream pipeline uses it.
    subtract_sky(tempframe,skymodel) #- Note: sky subtract is done to get residuals. As part of pipeline it is done in fluxcalib stage

    qadict = qalib.sky_resid(param, tempframe, skymodel, quick_look=quick_look)

    qadict_snr = qalib.SignalVsNoise(tempframe,param)
    qadict.update(qadict_snr)

    return qadict

<|endoftext|>


<commit_before># Creates a Droplet on Digital Ocean for remote Zulip development.
# Particularly useful for sprints/hackathons, interns, and other
# situation where one wants to quickly onboard new contributors.
#
# This script takes one argument: the name of the GitHub user for whom you want
# to create a Zulip developer environment. Requires Python 3.
#
# Requires python-digitalocean library:
# https://github.com/koalalorenzo/python-digitalocean
#
# Also requires Digital Ocean team membership for Zulip and api token:
# https://cloud.digitalocean.com/settings/api/tokens
#
# Copy conf.ini-template to conf.ini and populate with your api token.
#
# usage: python3 create.py <username>

import sys
import configparser
import urllib.error
import urllib.request
import json
import digitalocean
import time
import argparse
import os

from typing import Any, Dict, List

# initiation argument parser
parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.')
parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet")
parser.add_argument('--tags', nargs='+', default=[])
parser.add_argument('-f', '--recreate', dest='recreate', action="store_true", default=False)

def get_config() -> configparser.ConfigParser:
    config = configparser.ConfigParser()
    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf.ini'))
    return config

def user_exists(username: str) -> bool:
    print("Checking to see if GitHub user {} exists...".format(username))
    user_api_url = "https://api.github.com/users/{}".format(username)
    try:
        response = urllib.request.urlopen(user_api_url)
        json.load(response)
        print("...user exists!")
        return True
    except urllib.error.HTTPError as err:
        print(err)
        print("Does the github user {} exist?".format(username))
        sys.exit(1)

def get_keys(username: str) -> List[Dict[str, Any]]:
    print("Checking to see that GitHub user has available public keys...")
    apiurl_keys = "https://api.github.com/users/{}/keys".format(username)
    try:
        response = urllib.request.urlopen(apiurl_keys)
        userkeys = json.load(response)
        if not userkeys:
            print("No keys found. Has user {} added ssh keys to their github account?".format(username))
            sys.exit(1)
        print("...public keys found!")
        return userkeys
    except urllib.error.HTTPError as err:
        print(err)
        print("Has user {} added ssh keys to their github account?".format(username))
        sys.exit(1)

def fork_exists(username: str) -> bool:
    print("Checking to see GitHub user has forked zulip/zulip...")
    apiurl_fork = "https://api.github.com/repos/{}/zulip".format(username)
    try:
        response = urllib.request.urlopen(apiurl_fork)
        json.load(response)
        print("...fork found!")
        return True
    except urllib.error.HTTPError as err:
        print(err)
        print("Has user {} forked zulip/zulip?".format(username))
        sys.exit(1)

def exit_if_droplet_exists(my_token: str, username: str, recreate: bool) -> None:
    print("Checking to see if droplet for {} already exists...".format(username))
    manager = digitalocean.Manager(token=my_token)
    my_droplets = manager.get_all_droplets()
    for droplet in my_droplets:
        if droplet.name == "{}.zulipdev.org".format(username):
            if not recreate:
                print("Droplet for user {} already exists. Pass --recreate if you "
                      "need to recreate the droplet.".format(username))
                sys.exit(1)
            else:
                print("Deleting existing droplet for {}.".format(username))
                droplet.destroy()
                return
    print("...No droplet found...proceeding.")

def set_user_data(username: str, userkey_dicts: List[Dict[str, Any]]) -> str:
    print("Setting cloud-config data, populated with GitHub user's public keys...")
    userkeys = [userkey_dict["key"] for userkey_dict in userkey_dicts]
    ssh_keys = "\n".join(userkeys)

    setup_root_ssh_keys = "printf '{keys}' > /root/.ssh/authorized_keys".format(keys=ssh_keys)
    setup_zulipdev_ssh_keys = "printf '{keys}' > /home/zulipdev/.ssh/authorized_keys".format(keys=ssh_keys)

    # We pass the hostname as username.zulipdev.org to the DigitalOcean API.
    # But some droplets (eg on 18.04) are created with with hostname set to just username.
    # So we fix the hostname using cloud-init.
    hostname_setup = "hostnamectl set-hostname {username}.zulipdev.org".format(username=username)

    setup_repo = (
        "cd /home/zulipdev/{1} && "
        "git remote add origin https://github.com/{0}/{1}.git && "
        "git fetch origin && "
        "git clean -f"
    )

    server_repo_setup = setup_repo.format(username, "zulip")
    python_api_repo_setup = setup_repo.format(username, "python-zulip-api")

    cloudconf = """\
#!/bin/bash

{setup_zulipdev_ssh_keys}
{setup_root_ssh_keys}
{hostname_setup}
su -c '{server_repo_setup}' zulipdev
su -c '{python_api_repo_setup}' zulipdev
su -c 'git config --global core.editor nano' zulipdev
su -c 'git config --global pull.rebase true' zulipdev
""".format(setup_root_ssh_keys=setup_root_ssh_keys,
           setup_zulipdev_ssh_keys=setup_zulipdev_ssh_keys,
           hostname_setup=hostname_setup,
           server_repo_setup=server_repo_setup, python_api_repo_setup=python_api_repo_setup)
    print("...returning cloud-config data.")
    return cloudconf

def create_droplet(my_token: str, template_id: str, username: str, tags: List[str], user_data: str) -> str:
    droplet = digitalocean.Droplet(
        token=my_token,
        name='{}.zulipdev.org'.format(username),
        region='nyc3',
        image=template_id,
        size_slug='s-1vcpu-2gb',
        user_data=user_data,
        tags=tags,
        backups=False)

    print("Initiating droplet creation...")
    droplet.create()

    incomplete = True
    while incomplete:
        actions = droplet.get_actions()
        for action in actions:
            action.load()
            print("...[{}]: {}".format(action.type, action.status))
            if action.type == 'create' and action.status == 'completed':
                incomplete = False
                break
        if incomplete:
            time.sleep(15)
    print("...droplet created!")
    droplet.load()
    print("...ip address for new droplet is: {}.".format(droplet.ip_address))
    return droplet.ip_address

def delete_existing_records(records: List[digitalocean.Record], record_name: str) -> None:
    count = 0
    for record in records:
        if record.name == record_name and record.domain == 'zulipdev.org' and record.type == 'A':
            record.destroy()
            count = count + 1
    if count:
        print("Deleted {} existing A records for {}.zulipdev.org.".format(count, record_name))

def create_dns_record(my_token: str, username: str, ip_address: str) -> None:
    domain = digitalocean.Domain(token=my_token, name='zulipdev.org')
    domain.load()
    records = domain.get_records()

    delete_existing_records(records, username)
    wildcard_name = "*." + username
    delete_existing_records(records, wildcard_name)

    print("Creating new A record for {}.zulipdev.org that points to {}.".format(username, ip_address))
    domain.create_new_domain_record(type='A', name=username, data=ip_address)
    print("Creating new A record for *.{}.zulipdev.org that points to {}.".format(username, ip_address))
    domain.create_new_domain_record(type='A', name=wildcard_name, data=ip_address)

def print_completion(username: str) -> None:
    print("""
COMPLETE! Droplet for GitHub user {0} is available at {0}.zulipdev.org.

Instructions for use are below. (copy and paste to the user)

------
Your remote Zulip dev server has been created!

- Connect to your server by running
  `ssh zulipdev@{0}.zulipdev.org` on the command line
  (Terminal for macOS and Linux, Bash for Git on Windows).
- There is no password; your account is configured to use your ssh keys.
- Once you log in, you should see `(zulip-py3-venv) ~$`.
- To start the dev server, `cd zulip` and then run `./tools/run-dev.py`.
- While the dev server is running, you can see the Zulip server in your browser at
  http://{0}.zulipdev.org:9991.
""".format(username))

    print("See [Developing remotely](https://zulip.readthedocs.io/en/latest/development/remote.html) "
          "for tips on using the remote dev instance and "
          "[Git & GitHub Guide](https://zulip.readthedocs.io/en/latest/git/index.html) "
          "to learn how to use Git with Zulip.\n")
    print("Note that this droplet will automatically be deleted after a month of inactivity. "
          "If you are leaving Zulip for more than a few weeks, we recommend pushing all of your "
          "active branches to GitHub.")
    print("------")

if __name__ == '__main__':
    # define id of image to create new droplets from
    # You can get this with something like the following. You may need to try other pages.
    # Broken in two to satisfy linter (line too long)
    # curl -X GET -H "Content-Type: application/json" -u <API_KEY>: "https://api.digitaloc
    # ean.com/v2/images?page=5" | grep --color=always base.zulipdev.org
    template_id = "63219191"

    # get command line arguments
    args = parser.parse_args()
    print("Creating Zulip developer environment for GitHub user {}...".format(args.username))

    # get config details
    config = get_config()

    # see if droplet already exists for this user
    user_exists(username=args.username)

    # grab user's public keys
    public_keys = get_keys(username=args.username)

    # now make sure the user has forked zulip/zulip
    fork_exists(username=args.username)

    api_token = config['digitalocean']['api_token']
    # does the droplet already exist?
    exit_if_droplet_exists(my_token=api_token, username=args.username, recreate=args.recreate)

    # set user_data
    user_data = set_user_data(username=args.username, userkey_dicts=public_keys)

    # create droplet
    ip_address = create_droplet(my_token=api_token,
                                template_id=template_id,
                                username=args.username,
                                tags=args.tags,
                                user_data=user_data)

    # create dns entry
    create_dns_record(my_token=api_token, username=args.username, ip_address=ip_address)

    # print completion message
    print_completion(username=args.username)

    sys.exit(1)
<commit_msg>droplets: Disable SSH password authentication.<commit_after># Creates a Droplet on Digital Ocean for remote Zulip development.
# Particularly useful for sprints/hackathons, interns, and other
# situation where one wants to quickly onboard new contributors.
#
# This script takes one argument: the name of the GitHub user for whom you want
# to create a Zulip developer environment. Requires Python 3.
#
# Requires python-digitalocean library:
# https://github.com/koalalorenzo/python-digitalocean
#
# Also requires Digital Ocean team membership for Zulip and api token:
# https://cloud.digitalocean.com/settings/api/tokens
#
# Copy conf.ini-template to conf.ini and populate with your api token.
#
# usage: python3 create.py <username>

import sys
import configparser
import urllib.error
import urllib.request
import json
import digitalocean
import time
import argparse
import os

from typing import Any, Dict, List

# initiation argument parser
parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.')
parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet")
parser.add_argument('--tags', nargs='+', default=[])
parser.add_argument('-f', '--recreate', dest='recreate', action="store_true", default=False)

def get_config() -> configparser.ConfigParser:
    config = configparser.ConfigParser()
    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'conf.ini'))
    return config

def user_exists(username: str) -> bool:
    print("Checking to see if GitHub user {} exists...".format(username))
    user_api_url = "https://api.github.com/users/{}".format(username)
    try:
        response = urllib.request.urlopen(user_api_url)
        json.load(response)
        print("...user exists!")
        return True
    except urllib.error.HTTPError as err:
        print(err)
        print("Does the github user {} exist?".format(username))
        sys.exit(1)

def get_keys(username: str) -> List[Dict[str, Any]]:
    print("Checking to see that GitHub user has available public keys...")
    apiurl_keys = "https://api.github.com/users/{}/keys".format(username)
    try:
        response = urllib.request.urlopen(apiurl_keys)
        userkeys = json.load(response)
        if not userkeys:
            print("No keys found. Has user {} added ssh keys to their github account?".format(username))
            sys.exit(1)
        print("...public keys found!")
        return userkeys
    except urllib.error.HTTPError as err:
        print(err)
        print("Has user {} added ssh keys to their github account?".format(username))
        sys.exit(1)

def fork_exists(username: str) -> bool:
    print("Checking to see GitHub user has forked zulip/zulip...")
    apiurl_fork = "https://api.github.com/repos/{}/zulip".format(username)
    try:
        response = urllib.request.urlopen(apiurl_fork)
        json.load(response)
        print("...fork found!")
        return True
    except urllib.error.HTTPError as err:
        print(err)
        print("Has user {} forked zulip/zulip?".format(username))
        sys.exit(1)

def exit_if_droplet_exists(my_token: str, username: str, recreate: bool) -> None:
    print("Checking to see if droplet for {} already exists...".format(username))
    manager = digitalocean.Manager(token=my_token)
    my_droplets = manager.get_all_droplets()
    for droplet in my_droplets:
        if droplet.name == "{}.zulipdev.org".format(username):
            if not recreate:
                print("Droplet for user {} already exists. Pass --recreate if you "
                      "need to recreate the droplet.".format(username))
                sys.exit(1)
            else:
                print("Deleting existing droplet for {}.".format(username))
                droplet.destroy()
                return
    print("...No droplet found...proceeding.")

def set_user_data(username: str, userkey_dicts: List[Dict[str, Any]]) -> str:
    print("Setting cloud-config data, populated with GitHub user's public keys...")
    userkeys = [userkey_dict["key"] for userkey_dict in userkey_dicts]
    ssh_keys = "\n".join(userkeys)

    setup_root_ssh_keys = "printf '{keys}' > /root/.ssh/authorized_keys".format(keys=ssh_keys)
    setup_zulipdev_ssh_keys = "printf '{keys}' > /home/zulipdev/.ssh/authorized_keys".format(keys=ssh_keys)

    # We pass the hostname as username.zulipdev.org to the DigitalOcean API.
    # But some droplets (eg on 18.04) are created with with hostname set to just username.
    # So we fix the hostname using cloud-init.
    hostname_setup = "hostnamectl set-hostname {username}.zulipdev.org".format(username=username)

    setup_repo = (
        "cd /home/zulipdev/{1} && "
        "git remote add origin https://github.com/{0}/{1}.git && "
        "git fetch origin && "
        "git clean -f"
    )

    server_repo_setup = setup_repo.format(username, "zulip")
    python_api_repo_setup = setup_repo.format(username, "python-zulip-api")

    cloudconf = """\
#!/bin/bash

{setup_zulipdev_ssh_keys}
{setup_root_ssh_keys}
sed -i "s/PasswordAuthentication yes/PasswordAuthentication no/g" /etc/ssh/sshd_config
service ssh restart
{hostname_setup}
su -c '{server_repo_setup}' zulipdev
su -c '{python_api_repo_setup}' zulipdev
su -c 'git config --global core.editor nano' zulipdev
su -c 'git config --global pull.rebase true' zulipdev
""".format(setup_root_ssh_keys=setup_root_ssh_keys,
           setup_zulipdev_ssh_keys=setup_zulipdev_ssh_keys,
           hostname_setup=hostname_setup,
           server_repo_setup=server_repo_setup, python_api_repo_setup=python_api_repo_setup)
    print("...returning cloud-config data.")
    return cloudconf

def create_droplet(my_token: str, template_id: str, username: str, tags: List[str], user_data: str) -> str:
    droplet = digitalocean.Droplet(
        token=my_token,
        name='{}.zulipdev.org'.format(username),
        region='nyc3',
        image=template_id,
        size_slug='s-1vcpu-2gb',
        user_data=user_data,
        tags=tags,
        backups=False)

    print("Initiating droplet creation...")
    droplet.create()

    incomplete = True
    while incomplete:
        actions = droplet.get_actions()
        for action in actions:
            action.load()
            print("...[{}]: {}".format(action.type, action.status))
            if action.type == 'create' and action.status == 'completed':
                incomplete = False
                break
        if incomplete:
            time.sleep(15)
    print("...droplet created!")
    droplet.load()
    print("...ip address for new droplet is: {}.".format(droplet.ip_address))
    return droplet.ip_address

def delete_existing_records(records: List[digitalocean.Record], record_name: str) -> None:
    count = 0
    for record in records:
        if record.name == record_name and record.domain == 'zulipdev.org' and record.type == 'A':
            record.destroy()
            count = count + 1
    if count:
        print("Deleted {} existing A records for {}.zulipdev.org.".format(count, record_name))

def create_dns_record(my_token: str, username: str, ip_address: str) -> None:
    domain = digitalocean.Domain(token=my_token, name='zulipdev.org')
    domain.load()
    records = domain.get_records()

    delete_existing_records(records, username)
    wildcard_name = "*." + username
    delete_existing_records(records, wildcard_name)

    print("Creating new A record for {}.zulipdev.org that points to {}.".format(username, ip_address))
    domain.create_new_domain_record(type='A', name=username, data=ip_address)
    print("Creating new A record for *.{}.zulipdev.org that points to {}.".format(username, ip_address))
    domain.create_new_domain_record(type='A', name=wildcard_name, data=ip_address)

def print_completion(username: str) -> None:
    print("""
COMPLETE! Droplet for GitHub user {0} is available at {0}.zulipdev.org.

Instructions for use are below. (copy and paste to the user)

------
Your remote Zulip dev server has been created!

- Connect to your server by running
  `ssh zulipdev@{0}.zulipdev.org` on the command line
  (Terminal for macOS and Linux, Bash for Git on Windows).
- There is no password; your account is configured to use your ssh keys.
- Once you log in, you should see `(zulip-py3-venv) ~$`.
- To start the dev server, `cd zulip` and then run `./tools/run-dev.py`.
- While the dev server is running, you can see the Zulip server in your browser at
  http://{0}.zulipdev.org:9991.
""".format(username))

    print("See [Developing remotely](https://zulip.readthedocs.io/en/latest/development/remote.html) "
          "for tips on using the remote dev instance and "
          "[Git & GitHub Guide](https://zulip.readthedocs.io/en/latest/git/index.html) "
          "to learn how to use Git with Zulip.\n")
    print("Note that this droplet will automatically be deleted after a month of inactivity. "
          "If you are leaving Zulip for more than a few weeks, we recommend pushing all of your "
          "active branches to GitHub.")
    print("------")

if __name__ == '__main__':
    # define id of image to create new droplets from
    # You can get this with something like the following. You may need to try other pages.
    # Broken in two to satisfy linter (line too long)
    # curl -X GET -H "Content-Type: application/json" -u <API_KEY>: "https://api.digitaloc
    # ean.com/v2/images?page=5" | grep --color=always base.zulipdev.org
    template_id = "63219191"

    # get command line arguments
    args = parser.parse_args()
    print("Creating Zulip developer environment for GitHub user {}...".format(args.username))

    # get config details
    config = get_config()

    # see if droplet already exists for this user
    user_exists(username=args.username)

    # grab user's public keys
    public_keys = get_keys(username=args.username)

    # now make sure the user has forked zulip/zulip
    fork_exists(username=args.username)

    api_token = config['digitalocean']['api_token']
    # does the droplet already exist?
    exit_if_droplet_exists(my_token=api_token, username=args.username, recreate=args.recreate)

    # set user_data
    user_data = set_user_data(username=args.username, userkey_dicts=public_keys)

    # create droplet
    ip_address = create_droplet(my_token=api_token,
                                template_id=template_id,
                                username=args.username,
                                tags=args.tags,
                                user_data=user_data)

    # create dns entry
    create_dns_record(my_token=api_token, username=args.username, ip_address=ip_address)

    # print completion message
    print_completion(username=args.username)

    sys.exit(1)
<|endoftext|>