<commit_before>from deepchem.feat.base_classes import MolecularFeaturizer
from deepchem.utils.molecule_feature_utils import one_hot_encode
from deepchem.utils.typing import RDKitMol, RDKitAtom
import numpy as np
from typing import Tuple, Any
from dataclasses import dataclass

@dataclass
class MATEncoding:
  """
  Dataclass specific to the Molecular Attention Transformer [1]_.

  This dataclass class wraps around three different matrices for a given molecule: Node Features, Adjacency Matrix, and the Distance Matrix.

  Parameters
  ----------
  node_features: np.ndarray
    Node Features matrix for the molecule. For MAT, derived from the construct_node_features_matrix function.
  adjacency_matrix: np.ndarray
    Adjacency matrix for the molecule. Derived from rdkit.Chem.rdmolops.GetAdjacencyMatrix
  distance_matrix: np.ndarray
    Distance matrix for the molecule. Derived from rdkit.Chem.rdmolops.GetDistanceMatrix
  """
  node_features: np.ndarray
  adjacency_matrix: np.ndarray
  distance_matrix: np.ndarray

class MATFeaturizer(MolecularFeaturizer):
  """
  This class is a featurizer for the Molecule Attention Transformer [1]_.
  The returned value is a numpy array which consists of molecular graph descriptions:
    - Node Features
    - Adjacency Matrix
    - Distance Matrix

  References
  ---------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer`<https://arxiv.org/abs/2002.08264>`"

  Examples
  --------
  >>> import deepchem as dc
  >>> feat = dc.feat.MATFeaturizer()
  >>> out = feat.featurize("CCC")

  Note
  ----
  This class requires RDKit to be installed.
  """

  def __init__(self):
    pass

  def construct_mol(self, mol: RDKitMol) -> RDKitMol:
    """
    Processes an input RDKitMol further to be able to extract id-specific Conformers from it using mol.GetConformer().

    Parameters
    ----------
    mol: RDKitMol
      RDKit Mol object.

    Returns
    ----------
    mol: RDKitMol
      A processed RDKitMol objeect which is embedded, UFF Optimized and has Hydrogen atoms removed. If the former conditions are not met and there is a value error, then 2D Coordinates are computed instead.
    """
    try:
      from rdkit.Chem import AllChem
      from rdkit import Chem
    except:
      raise ModuleNotFoundError('This class requires RDKit to be installed.')
    try:
      mol = Chem.AddHs(mol)
      AllChem.EmbedMolecule(mol, maxAttempts=5000)
      AllChem.UFFOptimizeMolecule(mol)
      mol = Chem.RemoveHs(mol)
    except ValueError:
      AllChem.Compute2DCoords(mol)

    return mol

  def atom_features(self, atom: RDKitAtom) -> np.ndarray:
    """
    Deepchem already contains an atom_features function, however we are defining a new one here due to the need to handle features specific to MAT.
    Since we need new features like Atom GetNeighbors and IsInRing, and the number of features required for MAT is a fraction of what the Deepchem atom_features function computes, we can speed up computation by defining a custom function.

    Parameters
    ----------
    atom: RDKitAtom
      RDKit Atom object.

    Returns
    ----------
    Atom_features: ndarray
      Numpy array containing atom features.
    """
    attrib = []
    attrib += one_hot_encode(atom.GetAtomicNum(),
                             [5, 6, 7, 8, 9, 15, 16, 17, 35, 53, 999])
    attrib += one_hot_encode(len(atom.GetNeighbors()), [0, 1, 2, 3, 4, 5])
    attrib += one_hot_encode(atom.GetTotalNumHs(), [0, 1, 2, 3, 4])

    attrib += one_hot_encode(atom.GetFormalCharge(),
                             [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5])

    attrib.append(atom.IsInRing())
    attrib.append(atom.GetIsAromatic())

    return np.array(attrib, dtype=np.float32)

  def construct_node_features_matrix(self, mol: RDKitMol) -> np.ndarray:
    """
    This function constructs a matrix of atom features for all atoms in a given molecule using the atom_features function.

    Parameters
    ----------
    mol: RDKitMol
      RDKit Mol object.

    Returns
    ----------
    Atom_features: ndarray
      Numpy array containing atom features.
    """
    return np.array([self.atom_features(atom) for atom in mol.GetAtoms()])

  def _add_dummy_node(
      self, node_features: np.ndarray, adj_matrix: np.ndarray,
      dist_matrix: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Adds a single dummy node to the molecule, which is consequently reflected in the Node Features Matrix, Adjacency Matrix and the Distance Matrix.

    Parameters
    ----------
    node_features: np.ndarray
      Node Features matrix for a given molecule.
    adjacency_matrix: np.ndarray
      Adjacency matrix for a given molecule.
    distance_matrix: np.ndarray
      Distance matrix for a given molecule.

    Returns
    ----------
    Atom_features: Tuple[np.ndarray, np.ndarray, np.ndarray]
      A tuple containing three numpy arrays: node_features, adjacency_matrix, distance_matrix.
    """

    if node_features is not None:
      m = np.zeros((node_features.shape[0] + 1, node_features.shape[1] + 1))
      m[1:, 1:] = node_features
      m[0, 0] = 1.0
      node_features = m

    if adj_matrix is not None:
      m = np.zeros((adj_matrix.shape[0] + 1, adj_matrix.shape[1] + 1))
      m[1:, 1:] = adj_matrix
      adj_matrix = m

    if dist_matrix is not None:
      m = np.full((dist_matrix.shape[0] + 1, dist_matrix.shape[1] + 1), 1e6)
      m[1:, 1:] = dist_matrix
      dist_matrix = m

    return node_features, adj_matrix, dist_matrix

  def _pad_array(self, array: np.ndarray, shape: Any) -> np.ndarray:
    """
    Pads an array to the desired shape.

    Parameters
    ----------
    array: np.ndarray
      Array to be padded.
    shape: int or Tuple
      Shape the array is padded to.

    Returns
    ----------
    array: np.ndarray
      Array padded to input shape.
    """
    result = np.zeros(shape=shape)
    slices = tuple(slice(s) for s in array.shape)
    result[slices] = array
    return result

  def _pad_sequence(self, sequence: np.ndarray) -> np.ndarray:
    """
    Pads a given sequence using the pad_array function.

    Parameters
    ----------
    sequence: np.ndarray
      Arrays in this sequence are padded to the largest shape in the sequence.

    Returns
    ----------
    array: np.ndarray
      Sequence with padded arrays.
    """
    shapes = np.stack([np.array(t.shape) for t in sequence])
    max_shape = tuple(np.max(shapes, axis=0))
    return np.stack([self._pad_array(t, shape=max_shape) for t in sequence])

  def _featurize(self, datapoint: RDKitMol, **kwargs) -> np.ndarray:
    """
    Featurize the molecule.

    Parameters
    ----------
    datapoint: RDKitMol
      RDKit mol object.

    Returns
    -------
    MATEncoding: A MATEncoding dataclass instance consisting of processed node_features, adjacency_matrix and distance_matrix.
    """
    if 'mol' in kwargs:
      datapoint = kwargs.get("mol")
      raise DeprecationWarning(
          'Mol is being phased out as a parameter, please pass "datapoint" instead.'
      )

    try:
      from rdkit import Chem
    except:
      raise ImportError("This class requires RDKit to be installed.")

    datapoint = self.construct_mol(datapoint)

    node_features = self.construct_node_features_matrix(datapoint)
    adjacency_matrix = Chem.GetAdjacencyMatrix(datapoint)
    distance_matrix = Chem.GetDistanceMatrix(datapoint)

    node_features, adjacency_matrix, distance_matrix = self._add_dummy_node(
        node_features, adjacency_matrix, distance_matrix)

    node_features = self._pad_sequence(node_features)
    adjacency_matrix = self._pad_sequence(adjacency_matrix)
    distance_matrix = self._pad_sequence(distance_matrix)

    return MATEncoding(node_features, adjacency_matrix, distance_matrix)
<commit_msg>Addressed review comments<commit_after>from deepchem.feat.base_classes import MolecularFeaturizer
from deepchem.utils.molecule_feature_utils import one_hot_encode
from deepchem.utils.typing import RDKitMol, RDKitAtom
import numpy as np
from typing import Tuple, Any
from dataclasses import dataclass

@dataclass
class MATEncoding:
  """
  Dataclass specific to the Molecular Attention Transformer [1]_.

  This dataclass class wraps around three different matrices for a given molecule: Node Features, Adjacency Matrix, and the Distance Matrix.

  Parameters
  ----------
  node_features: np.ndarray
    Node Features matrix for the molecule. For MAT, derived from the construct_node_features_matrix function.
  adjacency_matrix: np.ndarray
    Adjacency matrix for the molecule. Derived from rdkit.Chem.rdmolops.GetAdjacencyMatrix
  distance_matrix: np.ndarray
    Distance matrix for the molecule. Derived from rdkit.Chem.rdmolops.GetDistanceMatrix
  """
  node_features: np.ndarray
  adjacency_matrix: np.ndarray
  distance_matrix: np.ndarray

class MATFeaturizer(MolecularFeaturizer):
  """
  This class is a featurizer for the Molecule Attention Transformer [1]_.
  The returned value is a numpy array which consists of molecular graph descriptions:
    - Node Features
    - Adjacency Matrix
    - Distance Matrix

  References
  ---------
  .. [1] Lukasz Maziarka et al. "Molecule Attention Transformer`<https://arxiv.org/abs/2002.08264>`"

  Examples
  --------
  >>> import deepchem as dc
  >>> feat = dc.feat.MATFeaturizer()
  >>> out = feat.featurize("CCC")

  Note
  ----
  This class requires RDKit to be installed.
  """

  def __init__(self):
    pass

  def construct_mol(self, mol: RDKitMol) -> RDKitMol:
    """
    Processes an input RDKitMol further to be able to extract id-specific Conformers from it using mol.GetConformer().

    Parameters
    ----------
    mol: RDKitMol
      RDKit Mol object.

    Returns
    ----------
    mol: RDKitMol
      A processed RDKitMol objeect which is embedded, UFF Optimized and has Hydrogen atoms removed. If the former conditions are not met and there is a value error, then 2D Coordinates are computed instead.
    """
    try:
      from rdkit.Chem import AllChem
      from rdkit import Chem
    except ModuleNotFoundError:
      pass
    try:
      mol = Chem.AddHs(mol)
      AllChem.EmbedMolecule(mol, maxAttempts=5000)
      AllChem.UFFOptimizeMolecule(mol)
      mol = Chem.RemoveHs(mol)
    except ValueError:
      AllChem.Compute2DCoords(mol)

    return mol

  def atom_features(self, atom: RDKitAtom) -> np.ndarray:
    """
    Deepchem already contains an atom_features function, however we are defining a new one here due to the need to handle features specific to MAT.
    Since we need new features like Atom GetNeighbors and IsInRing, and the number of features required for MAT is a fraction of what the Deepchem atom_features function computes, we can speed up computation by defining a custom function.

    Parameters
    ----------
    atom: RDKitAtom
      RDKit Atom object.

    Returns
    ----------
    Atom_features: ndarray
      Numpy array containing atom features.
    """
    attrib = []
    attrib += one_hot_encode(atom.GetAtomicNum(),
                             [5, 6, 7, 8, 9, 15, 16, 17, 35, 53, 999])
    attrib += one_hot_encode(len(atom.GetNeighbors()), [0, 1, 2, 3, 4, 5])
    attrib += one_hot_encode(atom.GetTotalNumHs(), [0, 1, 2, 3, 4])

    attrib += one_hot_encode(atom.GetFormalCharge(),
                             [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5])

    attrib.append(atom.IsInRing())
    attrib.append(atom.GetIsAromatic())

    return np.array(attrib, dtype=np.float32)

  def construct_node_features_matrix(self, mol: RDKitMol) -> np.ndarray:
    """
    This function constructs a matrix of atom features for all atoms in a given molecule using the atom_features function.

    Parameters
    ----------
    mol: RDKitMol
      RDKit Mol object.

    Returns
    ----------
    Atom_features: ndarray
      Numpy array containing atom features.
    """
    return np.array([self.atom_features(atom) for atom in mol.GetAtoms()])

  def _add_dummy_node(
      self, node_features: np.ndarray, adj_matrix: np.ndarray,
      dist_matrix: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Adds a single dummy node to the molecule, which is consequently reflected in the Node Features Matrix, Adjacency Matrix and the Distance Matrix.

    Parameters
    ----------
    node_features: np.ndarray
      Node Features matrix for a given molecule.
    adjacency_matrix: np.ndarray
      Adjacency matrix for a given molecule.
    distance_matrix: np.ndarray
      Distance matrix for a given molecule.

    Returns
    ----------
    Atom_features: Tuple[np.ndarray, np.ndarray, np.ndarray]
      A tuple containing three numpy arrays: node_features, adjacency_matrix, distance_matrix.
    """

    if node_features is not None:
      m = np.zeros((node_features.shape[0] + 1, node_features.shape[1] + 1))
      m[1:, 1:] = node_features
      m[0, 0] = 1.0
      node_features = m

    if adj_matrix is not None:
      m = np.zeros((adj_matrix.shape[0] + 1, adj_matrix.shape[1] + 1))
      m[1:, 1:] = adj_matrix
      adj_matrix = m

    if dist_matrix is not None:
      m = np.full((dist_matrix.shape[0] + 1, dist_matrix.shape[1] + 1), 1e6)
      m[1:, 1:] = dist_matrix
      dist_matrix = m

    return node_features, adj_matrix, dist_matrix

  def _pad_array(self, array: np.ndarray, shape: Any) -> np.ndarray:
    """
    Pads an array to the desired shape.

    Parameters
    ----------
    array: np.ndarray
      Array to be padded.
    shape: int or Tuple
      Shape the array is padded to.

    Returns
    ----------
    array: np.ndarray
      Array padded to input shape.
    """
    result = np.zeros(shape=shape)
    slices = tuple(slice(s) for s in array.shape)
    result[slices] = array
    return result

  def _pad_sequence(self, sequence: np.ndarray) -> np.ndarray:
    """
    Pads a given sequence using the pad_array function.

    Parameters
    ----------
    sequence: np.ndarray
      Arrays in this sequence are padded to the largest shape in the sequence.

    Returns
    ----------
    array: np.ndarray
      Sequence with padded arrays.
    """
    shapes = np.stack([np.array(t.shape) for t in sequence])
    max_shape = tuple(np.max(shapes, axis=0))
    return np.stack([self._pad_array(t, shape=max_shape) for t in sequence])

  def _featurize(self, datapoint: RDKitMol, **kwargs) -> np.ndarray:
    """
    Featurize the molecule.

    Parameters
    ----------
    datapoint: RDKitMol
      RDKit mol object.

    Returns
    -------
    MATEncoding: A MATEncoding dataclass instance consisting of processed node_features, adjacency_matrix and distance_matrix.
    """
    if 'mol' in kwargs:
      datapoint = kwargs.get("mol")
      raise DeprecationWarning(
          'Mol is being phased out as a parameter, please pass "datapoint" instead.'
      )
    from rdkit import Chem

    datapoint = self.construct_mol(datapoint)

    node_features = self.construct_node_features_matrix(datapoint)
    adjacency_matrix = Chem.GetAdjacencyMatrix(datapoint)
    distance_matrix = Chem.GetDistanceMatrix(datapoint)

    node_features, adjacency_matrix, distance_matrix = self._add_dummy_node(
        node_features, adjacency_matrix, distance_matrix)

    node_features = self._pad_sequence(node_features)
    adjacency_matrix = self._pad_sequence(adjacency_matrix)
    distance_matrix = self._pad_sequence(distance_matrix)

    return MATEncoding(node_features, adjacency_matrix, distance_matrix)
<|endoftext|>


<commit_before>#!/usr/bin/env python2

""" Python 2.6 based package manager. """

import optparse
import json
import os
import sys
import stat
import subprocess
import tarfile
import shutil
from urllib2 import urlopen, HTTPError
from urlparse import urljoin
import re

_emptyConfig = {'packageManagerDir': '~/local/packageManager',
        'packageRepositoryURL': 'http://',
        'installationEnvironmentVariables': {'INSTALL_PREFIX': '~/local'},
        }

_availablePackagesDir = 'availablePackages'
_installedPackagesDir = 'installedPackages'
_sourcesDir = 'sources'
_installScriptsDir = 'installScripts'
_buildDir = 'build'

class PackageError(StandardError): pass
class PackageManagerError(StandardError): pass

class Version(object):
    def __init__(self, s):
        self._versionStr = s

    @classmethod
    def fromStr(cls, s):
        cls(s)

    def __str__(self):
        return self._versionStr

    def __eq__(self, other):
        if not isinstance(other, Version):
            return NotImplemented
        return self._versionStr == other._versionStr

    def __gt__(self, other):
        if not isinstance(other, Version):
            return NotImplemented
        # TODO real version comparison
        return self._versionStr != other._versionStr

class Package(object):
    def __init__(self, config):
        self.name = config['name']
        self.version = Version(config['version'])
        self.packageType = config['type']
        self.dependencies = config['dependencies']
        if self.packageType == 'archive':
            self.sourceFile = config['sourceFile']
            self.installScript = config['installScript']
        elif self.packageType == 'meta':
            pass
        elif self.packageType == 'git':
            # TODO
            pass
        else:
            raise ValueError("unknown package type {0}".format(self.packageType))

    def __str__(self):
        return '{0} ({1})'.format(self.name, self.version)

    def __eq__(self, other):
        if not isinstance(other, Package):
            return NotImplemented
        return (self.name == other.name and self.version == other.version)

    def __ne__(self, other):
        return not self.__eq__(other)

    @classmethod
    def fromJsonFile(cls, jsonFile):
        with open(jsonFile) as f:
            obj = cls(json.load(f))
            obj.configFile = os.path.basename(jsonFile)
            return obj

    def _unpackSource(self, sourcesPath, buildPath):
        sourceFile = os.path.join(sourcesPath, self.sourceFile)
        unpackTo = os.path.join(buildPath, self.name)
        if os.path.isdir(unpackTo):
            shutil.rmtree(unpackTo)
        os.makedirs(unpackTo)
        try:
            tf = tarfile.open(sourceFile)
        except OSError as e:
            raise PackageError("Error while unpacking source file: {0}".format(sourceFile))
        try:
            tf.extractall(path=unpackTo)
        finally:
            tf.close()

    def _runInstallScript(self, buildPath, installScriptsPath, environmentVariables):
        installScript = os.path.abspath(os.path.join(installScriptsPath, self.installScript))
        unpackedSource = os.path.join(buildPath, self.name)
        try:
            subprocess.check_call([installScript], cwd=unpackedSource, env=environmentVariables)
        except OSError as e:
            raise PackageError("Can not call installation script: {0}".format(e))
        except subprocess.CalledProcessError as e:
            raise PackageError("Error in installation script: {0}".format(e))

    def install(self, sourcesPath, installScriptsPath, buildPath, environmentVariables):
        print "\n--- installing {0} ---".format(self)
        if self.packageType == 'archive':
            self._unpackSource(sourcesPath, buildPath)
            self._runInstallScript(buildPath, installScriptsPath, environmentVariables)
        elif self.packageType == 'meta':
            pass
        elif self.packageType == 'git':
            # TODO
            raise NotImplementedError
        else:
            raise ValueError("unknown package type {0}".format(self.packageType))

class PackageManager(object):
    def __init__(self, config):
        self._basePath = os.path.expanduser(config['packageManagerDir'])
        self._availablePackagesPath = os.path.join(self._basePath, _availablePackagesDir)
        self._installedPackagesPath = os.path.join(self._basePath, _installedPackagesDir)
        self._sourcesPath = os.path.join(self._basePath, _sourcesDir)
        self._installScriptsPath = os.path.join(self._basePath, _installScriptsDir)
        self._buildPath = os.path.join(self._basePath, _buildDir)

        # create directories if they don't exist
        for d in [self._availablePackagesPath, self._installedPackagesPath, \
                self._sourcesPath, self._installScriptsPath, self._buildPath]:
            if not os.path.isdir(d):
                os.makedirs(d)

        self._installEnvs = config['installationEnvironmentVariables']
        self.packageRepoURL = config['packageRepositoryURL']

        self._availablePackages = self._readPackageConfigs(self._availablePackagesPath)
        self._installedPackages = self._readPackageConfigs(self._installedPackagesPath)

    def _readPackageConfigs(self, configsPath):
        packages = {}
        for packageConfigFile in os.listdir(configsPath):
            if not packageConfigFile.startswith('.'):
                p = Package.fromJsonFile(os.path.join(configsPath, packageConfigFile))
                packages[p.name] = p
        return packages

    def _getDependencies(self, package):
        allDependencies = []
        for dependencyName in package.dependencies:
            try:
                dependency = self._availablePackages[dependencyName]
            except KeyError:
                raise PackageManagerError("Can not resolve dependency '{0}' of package {1}"\
                        .format(dependencyName, package))
            allDependencies.append(dependency)
            allDependencies += self._getDependencies(dependency)
        return allDependencies

    def getAvailablePackages(self):
        return self._availablePackages

    def getInstalledPackages(self):
        return self._installedPackages

    def installPackages(self, packageNames):
        packagesToInstall = []
        for packageName in packageNames:
            if packageName not in self._availablePackages.keys():
                print "Package '{0}' is not available".format(packageName)
                continue
            package = self._availablePackages[packageName]
            if packageName in self._installedPackages.keys():
                installedPackage = self._installedPackages[packageName]
                if not package.version > installedPackage.version:
                    print "Package '{0.name}' is already installed in version {0.version}. " \
                            "No newer version is available.".format(self._installedPackages[packageName])
                    continue
            packageDependencies = [package] + self._getDependencies(package)
            for p in reversed(packageDependencies):
                if p not in self._installedPackages.values() and p not in packagesToInstall:
                    packagesToInstall = [p] + packagesToInstall

        if packagesToInstall:
            print "The following packages will be installed to satisfy dependencies:"
            for s in packagesToInstall:
                print "  {0}".format(s)
            print
            doIt = raw_input("are you sure? [y/n]\n")
            if doIt.lower() == 'y':
                print
                print "=== downloading packages ==="
                print
                self.downloadPackages(packagesToInstall)
                print
                print "=== installing packages ==="
                for p in reversed(packagesToInstall):
                    # install package
                    try:
                        p.install(self._sourcesPath, self._installScriptsPath, self._buildPath, \
                                self._installEnvs)
                    except PackageError as e:
                        raise PackageManagerError("Error while installing {0}: {1}".format(p, e))
                    # copy config file to installed packages directory
                    fromPath = os.path.join(self._availablePackagesPath, p.configFile)
                    toPath = os.path.join(self._installedPackagesPath, p.configFile)
                    shutil.copy(fromPath, toPath)

    def _downloadFile(self, url, destination, executable=False):
        try:
            remote = urlopen(url)
        except HTTPError as e:
            raise PackageManagerError("can not download {0}: {1}".format(url, e))
        try:
            with open(destination, 'w') as local:
                local.write(remote.read())
            if executable:
                st = os.stat(destination)
                os.chmod(destination, st.st_mode | stat.S_IEXEC)
        finally:
            remote.close()

    def downloadPackages(self, packages):
        sourcesDirURL = urljoin(self.packageRepoURL, _sourcesDir + '/')
        installScriptsDirURL = urljoin(self.packageRepoURL, _installScriptsDir + '/')
        for package in packages:
            # source file
            try:
                sourceFile = package.sourceFile
                print "downloading sources of {0}".format(package)
                sourceFileURL = urljoin(sourcesDirURL, sourceFile)
                localSourceFilePath = os.path.join(self._sourcesPath, sourceFile)
                self._downloadFile(sourceFileURL, localSourceFilePath)
            except AttributeError:
                pass
            # install script
            try:
                installScript = package.installScript
                print "downloading install script of {0}".format(package)
                installScriptURL = urljoin(installScriptsDirURL, installScript)
                localScriptPath = os.path.join(self._installScriptsPath, installScript)
                self._downloadFile(installScriptURL, localScriptPath, executable=True)
            except AttributeError:
                pass

    def updateAvailablePackages(self):
        print "Updating available packages repository"
        availablePackagesURL = urljoin(self.packageRepoURL, _availablePackagesDir + '/')
        availablePackagesPath = urlopen(availablePackagesURL)
        availablePackagesHTML = availablePackagesPath.read().decode('utf-8')
        packageFilePattern = re.compile('".*\.json"')

        packageFileList = map(lambda p: p.strip('"'), packageFilePattern.findall(availablePackagesHTML))
        for packageFileName in packageFileList:
            print "downloading {0}".format(packageFileName)
            packageFileURL = urljoin(availablePackagesURL, packageFileName)
            localFile = os.path.join(self._availablePackagesPath, packageFileName)
            self._downloadFile(packageFileURL, localFile)

        print "Done"

    def upgradeInstalledPackages(self):
        raise NotImplementedError
        for packageName in self._installedPackages:
            installedPackage = self._installedPackages[packageName]
            availablePackage = self._availablePackages[packageName]
            if availablePackage.version > installedPackage.version:
                availablePackage.install(self._sourcesPath, self._installScriptsPath, self._buildPath)
                # check for installed packages that depend on it and need to be recompiled
                for p in self._installedPackages:
                    if packageName in p.dependencies:
                        p.install(self._sourcesPath, self._installScriptsPath, self._buildPath)

def main():
    supportedCommands = ['install', 'update', 'upgrade']
    usage = "usage: %prog [options] {{{0}}} [package [package] ...]"\
            .format(', '.join(supportedCommands))
    optParser = optparse.OptionParser(usage=usage, description=__doc__)
    optParser.add_option('-c', '--config', dest='config', default='~/.pmconfig.json', \
            help="The package manager configuration file (default %default)")
    (opts, args) = optParser.parse_args()

    try:
        command = args[0]
        packages = args[1:]
    except IndexError:
        optParser.error("No command specified")

    # read config
    configFile = os.path.expanduser(opts.config)
    if not os.path.isfile(configFile):
        # config file doesn't exists, write template and exit
        print "No package manager config found at '{0}'. I have written a template. " \
                "Please edit it at '{0}'.".format(configFile)
        with open(configFile, 'w') as f:
            json.dump(_emptyConfig, f, sort_keys=True, indent=4, separators=(',', ': '))
        sys.exit(1)
    with open(configFile) as f:
        pmConfig = json.load(f)

    pm = PackageManager(pmConfig)

    if command == 'install':
        if packages:
            try:
                pm.installPackages(packages)
            except PackageManagerError as e:
                print >> sys.stderr, e
                sys.exit(-1)
        else:
            optParser.error("No packages to install provided")
    elif command == 'update':
        pm.updateAvailablePackages()
    elif command == 'upgrade':
        pm.upgradeInstalledPackages()

    else:
        optParser.error("Unsupported command: {0}".format(command))

if __name__ == '__main__':
    main()
<commit_msg>more robust re for finding configs in server listing<commit_after>#!/usr/bin/env python2

""" Python 2.6 based package manager. """

import optparse
import json
import os
import sys
import stat
import subprocess
import tarfile
import shutil
from urllib2 import urlopen, HTTPError
from urlparse import urljoin
import re

_emptyConfig = {'packageManagerDir': '~/local/packageManager',
        'packageRepositoryURL': 'http://',
        'installationEnvironmentVariables': {'INSTALL_PREFIX': '~/local'},
        }

_availablePackagesDir = 'availablePackages'
_installedPackagesDir = 'installedPackages'
_sourcesDir = 'sources'
_installScriptsDir = 'installScripts'
_buildDir = 'build'

class PackageError(StandardError): pass
class PackageManagerError(StandardError): pass

class Version(object):
    def __init__(self, s):
        self._versionStr = s

    @classmethod
    def fromStr(cls, s):
        cls(s)

    def __str__(self):
        return self._versionStr

    def __eq__(self, other):
        if not isinstance(other, Version):
            return NotImplemented
        return self._versionStr == other._versionStr

    def __gt__(self, other):
        if not isinstance(other, Version):
            return NotImplemented
        # TODO real version comparison
        return self._versionStr != other._versionStr

class Package(object):
    def __init__(self, config):
        self.name = config['name']
        self.version = Version(config['version'])
        self.packageType = config['type']
        self.dependencies = config['dependencies']
        if self.packageType == 'archive':
            self.sourceFile = config['sourceFile']
            self.installScript = config['installScript']
        elif self.packageType == 'meta':
            pass
        elif self.packageType == 'git':
            # TODO
            pass
        else:
            raise ValueError("unknown package type {0}".format(self.packageType))

    def __str__(self):
        return '{0} ({1})'.format(self.name, self.version)

    def __eq__(self, other):
        if not isinstance(other, Package):
            return NotImplemented
        return (self.name == other.name and self.version == other.version)

    def __ne__(self, other):
        return not self.__eq__(other)

    @classmethod
    def fromJsonFile(cls, jsonFile):
        with open(jsonFile) as f:
            obj = cls(json.load(f))
            obj.configFile = os.path.basename(jsonFile)
            return obj

    def _unpackSource(self, sourcesPath, buildPath):
        sourceFile = os.path.join(sourcesPath, self.sourceFile)
        unpackTo = os.path.join(buildPath, self.name)
        if os.path.isdir(unpackTo):
            shutil.rmtree(unpackTo)
        os.makedirs(unpackTo)
        try:
            tf = tarfile.open(sourceFile)
        except OSError as e:
            raise PackageError("Error while unpacking source file: {0}".format(sourceFile))
        try:
            tf.extractall(path=unpackTo)
        finally:
            tf.close()

    def _runInstallScript(self, buildPath, installScriptsPath, environmentVariables):
        installScript = os.path.abspath(os.path.join(installScriptsPath, self.installScript))
        unpackedSource = os.path.join(buildPath, self.name)
        try:
            subprocess.check_call([installScript], cwd=unpackedSource, env=environmentVariables)
        except OSError as e:
            raise PackageError("Can not call installation script: {0}".format(e))
        except subprocess.CalledProcessError as e:
            raise PackageError("Error in installation script: {0}".format(e))

    def install(self, sourcesPath, installScriptsPath, buildPath, environmentVariables):
        print "\n--- installing {0} ---".format(self)
        if self.packageType == 'archive':
            self._unpackSource(sourcesPath, buildPath)
            self._runInstallScript(buildPath, installScriptsPath, environmentVariables)
        elif self.packageType == 'meta':
            pass
        elif self.packageType == 'git':
            # TODO
            raise NotImplementedError
        else:
            raise ValueError("unknown package type {0}".format(self.packageType))

class PackageManager(object):
    def __init__(self, config):
        self._basePath = os.path.expanduser(config['packageManagerDir'])
        self._availablePackagesPath = os.path.join(self._basePath, _availablePackagesDir)
        self._installedPackagesPath = os.path.join(self._basePath, _installedPackagesDir)
        self._sourcesPath = os.path.join(self._basePath, _sourcesDir)
        self._installScriptsPath = os.path.join(self._basePath, _installScriptsDir)
        self._buildPath = os.path.join(self._basePath, _buildDir)

        # create directories if they don't exist
        for d in [self._availablePackagesPath, self._installedPackagesPath, \
                self._sourcesPath, self._installScriptsPath, self._buildPath]:
            if not os.path.isdir(d):
                os.makedirs(d)

        self._installEnvs = config['installationEnvironmentVariables']
        self.packageRepoURL = config['packageRepositoryURL']

        self._availablePackages = self._readPackageConfigs(self._availablePackagesPath)
        self._installedPackages = self._readPackageConfigs(self._installedPackagesPath)

    def _readPackageConfigs(self, configsPath):
        packages = {}
        for packageConfigFile in os.listdir(configsPath):
            if not packageConfigFile.startswith('.'):
                p = Package.fromJsonFile(os.path.join(configsPath, packageConfigFile))
                packages[p.name] = p
        return packages

    def _getDependencies(self, package):
        allDependencies = []
        for dependencyName in package.dependencies:
            try:
                dependency = self._availablePackages[dependencyName]
            except KeyError:
                raise PackageManagerError("Can not resolve dependency '{0}' of package {1}"\
                        .format(dependencyName, package))
            allDependencies.append(dependency)
            allDependencies += self._getDependencies(dependency)
        return allDependencies

    def getAvailablePackages(self):
        return self._availablePackages

    def getInstalledPackages(self):
        return self._installedPackages

    def installPackages(self, packageNames):
        packagesToInstall = []
        for packageName in packageNames:
            if packageName not in self._availablePackages.keys():
                print "Package '{0}' is not available".format(packageName)
                continue
            package = self._availablePackages[packageName]
            if packageName in self._installedPackages.keys():
                installedPackage = self._installedPackages[packageName]
                if not package.version > installedPackage.version:
                    print "Package '{0.name}' is already installed in version {0.version}. " \
                            "No newer version is available.".format(self._installedPackages[packageName])
                    continue
            packageDependencies = [package] + self._getDependencies(package)
            for p in reversed(packageDependencies):
                if p not in self._installedPackages.values() and p not in packagesToInstall:
                    packagesToInstall = [p] + packagesToInstall

        if packagesToInstall:
            print "The following packages will be installed to satisfy dependencies:"
            for s in packagesToInstall:
                print "  {0}".format(s)
            print
            doIt = raw_input("are you sure? [y/n]\n")
            if doIt.lower() == 'y':
                print
                print "=== downloading packages ==="
                print
                self.downloadPackages(packagesToInstall)
                print
                print "=== installing packages ==="
                for p in reversed(packagesToInstall):
                    # install package
                    try:
                        p.install(self._sourcesPath, self._installScriptsPath, self._buildPath, \
                                self._installEnvs)
                    except PackageError as e:
                        raise PackageManagerError("Error while installing {0}: {1}".format(p, e))
                    # copy config file to installed packages directory
                    fromPath = os.path.join(self._availablePackagesPath, p.configFile)
                    toPath = os.path.join(self._installedPackagesPath, p.configFile)
                    shutil.copy(fromPath, toPath)

    def _downloadFile(self, url, destination, executable=False):
        try:
            remote = urlopen(url)
        except HTTPError as e:
            raise PackageManagerError("can not download {0}: {1}".format(url, e))
        try:
            with open(destination, 'w') as local:
                local.write(remote.read())
            if executable:
                st = os.stat(destination)
                os.chmod(destination, st.st_mode | stat.S_IEXEC)
        finally:
            remote.close()

    def downloadPackages(self, packages):
        sourcesDirURL = urljoin(self.packageRepoURL, _sourcesDir + '/')
        installScriptsDirURL = urljoin(self.packageRepoURL, _installScriptsDir + '/')
        for package in packages:
            # source file
            try:
                sourceFile = package.sourceFile
                print "downloading sources of {0}".format(package)
                sourceFileURL = urljoin(sourcesDirURL, sourceFile)
                localSourceFilePath = os.path.join(self._sourcesPath, sourceFile)
                self._downloadFile(sourceFileURL, localSourceFilePath)
            except AttributeError:
                pass
            # install script
            try:
                installScript = package.installScript
                print "downloading install script of {0}".format(package)
                installScriptURL = urljoin(installScriptsDirURL, installScript)
                localScriptPath = os.path.join(self._installScriptsPath, installScript)
                self._downloadFile(installScriptURL, localScriptPath, executable=True)
            except AttributeError:
                pass

    def updateAvailablePackages(self):
        print "Updating available packages repository"
        availablePackagesURL = urljoin(self.packageRepoURL, _availablePackagesDir + '/')
        availablePackagesPath = urlopen(availablePackagesURL)
        availablePackagesHTML = availablePackagesPath.read().decode('utf-8')
        packageFilePattern = re.compile('"[^"]+\.json"')
        packageFileList = map(lambda p: p.strip('"'), packageFilePattern.findall(availablePackagesHTML))
        for packageFileName in packageFileList:
            print "downloading {0}".format(packageFileName)
            packageFileURL = urljoin(availablePackagesURL, packageFileName)
            localFile = os.path.join(self._availablePackagesPath, packageFileName)
            self._downloadFile(packageFileURL, localFile)

        print "Done"

    def upgradeInstalledPackages(self):
        raise NotImplementedError
        for packageName in self._installedPackages:
            installedPackage = self._installedPackages[packageName]
            availablePackage = self._availablePackages[packageName]
            if availablePackage.version > installedPackage.version:
                availablePackage.install(self._sourcesPath, self._installScriptsPath, self._buildPath)
                # check for installed packages that depend on it and need to be recompiled
                for p in self._installedPackages:
                    if packageName in p.dependencies:
                        p.install(self._sourcesPath, self._installScriptsPath, self._buildPath)

def main():
    supportedCommands = ['install', 'update', 'upgrade']
    usage = "usage: %prog [options] {{{0}}} [package [package] ...]"\
            .format(', '.join(supportedCommands))
    optParser = optparse.OptionParser(usage=usage, description=__doc__)
    optParser.add_option('-c', '--config', dest='config', default='~/.pmconfig.json', \
            help="The package manager configuration file (default %default)")
    (opts, args) = optParser.parse_args()

    try:
        command = args[0]
        packages = args[1:]
    except IndexError:
        optParser.error("No command specified")

    # read config
    configFile = os.path.expanduser(opts.config)
    if not os.path.isfile(configFile):
        # config file doesn't exists, write template and exit
        print "No package manager config found at '{0}'. I have written a template. " \
                "Please edit it at '{0}'.".format(configFile)
        with open(configFile, 'w') as f:
            json.dump(_emptyConfig, f, sort_keys=True, indent=4, separators=(',', ': '))
        sys.exit(1)
    with open(configFile) as f:
        pmConfig = json.load(f)

    pm = PackageManager(pmConfig)

    if command == 'install':
        if packages:
            try:
                pm.installPackages(packages)
            except PackageManagerError as e:
                print >> sys.stderr, e
                sys.exit(-1)
        else:
            optParser.error("No packages to install provided")
    elif command == 'update':
        pm.updateAvailablePackages()
    elif command == 'upgrade':
        pm.upgradeInstalledPackages()

    else:
        optParser.error("Unsupported command: {0}".format(command))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import glob
import os

from moulinette import m18n
from yunohost.utils.error import YunohostError
from moulinette.utils.log import getActionLogger
from moulinette.utils.process import check_output, call_async_output
from moulinette.utils.filesystem import read_file, rm

from yunohost.tools import Migration, tools_update, tools_upgrade
from yunohost.app import unstable_apps
from yunohost.regenconf import manually_modified_files, _force_clear_hashes
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.utils.packages import (
    get_ynh_package_version,
    _list_upgradable_apt_packages,
)
from yunohost.services import _get_services, _save_services

logger = getActionLogger("yunohost.migration")

N_CURRENT_DEBIAN = 10
N_CURRENT_YUNOHOST = 4

N_NEXT_DEBAN = 11
N_NEXT_YUNOHOST = 11

class MyMigration(Migration):

    "Upgrade the system to Debian Bullseye and Yunohost 11.x"

    mode = "manual"

    def run(self):

        self.check_assertions()

        logger.info(m18n.n("migration_0021_start"))

        #
        # Add new apt .deb signing key
        #

        new_apt_key = "https://forge.yunohost.org/yunohost_bullseye.asc"
        check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -")

        #
        # Patch sources.list
        #
        logger.info(m18n.n("migration_0021_patching_sources_list"))
        self.patch_apt_sources_list()
        tools_update(target="system")

        # Tell libc6 it's okay to restart system stuff during the upgrade
        os.system(
            "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections"
        )

        # Don't send an email to root about the postgresql migration. It should be handled automatically after.
        os.system(
            "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections"
        )

        #
        # Patch yunohost conflicts
        #
        logger.info(m18n.n("migration_0021_patch_yunohost_conflicts"))

        self.patch_yunohost_conflicts()

        #
        # Specific tweaking to get rid of custom my.cnf and use debian's default one
        # (my.cnf is actually a symlink to mariadb.cnf)
        #

        _force_clear_hashes(["/etc/mysql/my.cnf"])
        rm("/etc/mysql/mariadb.cnf", force=True)
        rm("/etc/mysql/my.cnf", force=True)
        self.apt_install("mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'")

        #
        # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl
        #
        if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"):
            os.system("mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl")
            rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True)

        #
        # /home/yunohost.conf -> /var/cache/yunohost/regenconf
        #
        if os.path.exists("/home/yunohost.conf"):
            os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf")
            rm("/home/yunohost.conf", recursive=True, force=True)

        #
        # Main upgrade
        #
        logger.info(m18n.n("migration_0021_main_upgrade"))

        apps_packages = self.get_apps_equivs_packages()
        self.hold(apps_packages)
        tools_upgrade(target="system", allow_yunohost_upgrade=False)

        if self.debian_major_version() == N_CURRENT_DEBIAN:
            raise YunohostError("migration_0021_still_on_buster_after_main_upgrade")

        # Clean the mess
        logger.info(m18n.n("migration_0021_cleaning_up"))
        os.system("apt autoremove --assume-yes")
        os.system("apt clean --assume-yes")

        # Force add sury if it's not there yet
        # This is to solve some weird issue with php-common breaking php7.3-common,
        # hence breaking many php7.3-deps
        # hence triggering some dependency conflict (or foobar-ynh-deps uninstall)
        # Adding it there shouldnt be a big deal - Yunohost 11.x does add it
        # through its regen conf anyway.
        if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"):
            open("/etc/apt/sources.list.d/extra_php_version.list", "w").write("deb https://packages.sury.org/php/ bullseye main")
            os.system('wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"')

        os.system("apt update")

        # Force explicit install of php7.4-fpm to make sure it's ll be there
        # during 0022_php73_to_php74_pools migration
        self.apt_install("php7.4-fpm -o Dpkg::Options::='--force-confmiss'")

        # Remove legacy postgresql service record added by helpers,
        # will now be dynamically handled by the core in bullseye
        services = _get_services()
        if "postgresql" in services:
            del services["postgresql"]
            _save_services(services)

        #
        # Yunohost upgrade
        #
        logger.info(m18n.n("migration_0021_yunohost_upgrade"))
        self.unhold(apps_packages)
        tools_upgrade(target="system")

    def debian_major_version(self):
        # The python module "platform" and lsb_release are not reliable because
        # on some setup, they may still return Release=9 even after upgrading to
        # buster ... (Apparently this is related to OVH overriding some stuff
        # with /etc/lsb-release for instance -_-)
        # Instead, we rely on /etc/os-release which should be the raw info from
        # the distribution...
        return int(
            check_output(
                "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2"
            )
        )

    def yunohost_major_version(self):
        return int(get_ynh_package_version("yunohost")["version"].split(".")[0])

    def check_assertions(self):

        # Be on buster (10.x) and yunohost 4.x
        # NB : we do both check to cover situations where the upgrade crashed
        # in the middle and debian version could be > 9.x but yunohost package
        # would still be in 3.x...
        if (
            not self.debian_major_version() == N_CURRENT_DEBIAN
            and not self.yunohost_major_version() == N_CURRENT_YUNOHOST
        ):
            raise YunohostError("migration_0021_not_buster")

        # Have > 1 Go free space on /var/ ?
        if free_space_in_directory("/var/") / (1024 ** 3) < 1.0:
            raise YunohostError("migration_0021_not_enough_free_space")

        # Check system is up to date
        # (but we don't if 'bullseye' is already in the sources.list ...
        # which means maybe a previous upgrade crashed and we're re-running it)
        if " bullseye " not in read_file("/etc/apt/sources.list"):
            tools_update(target="system")
            upgradable_system_packages = list(_list_upgradable_apt_packages())
            if upgradable_system_packages:
                raise YunohostError("migration_0021_system_not_fully_up_to_date")

    @property
    def disclaimer(self):

        # Avoid having a super long disclaimer + uncessary check if we ain't
        # on buster / yunohost 4.x anymore
        # NB : we do both check to cover situations where the upgrade crashed
        # in the middle and debian version could be >= 10.x but yunohost package
        # would still be in 4.x...
        if (
            not self.debian_major_version() == N_CURRENT_DEBIAN
            and not self.yunohost_major_version() == N_CURRENT_YUNOHOST
        ):
            return None

        # Get list of problematic apps ? I.e. not official or community+working
        problematic_apps = unstable_apps()
        problematic_apps = "".join(["\n    - " + app for app in problematic_apps])

        # Manually modified files ? (c.f. yunohost service regen-conf)
        modified_files = manually_modified_files()
        modified_files = "".join(["\n    - " + f for f in modified_files])

        message = m18n.n("migration_0021_general_warning")

        # FIXME: re-enable this message with updated topic link once we release the migration as stable
        #message = (
        #    "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n"
        #    + message
        #)

        if problematic_apps:
            message += "\n\n" + m18n.n(
                "migration_0021_problematic_apps_warning",
                problematic_apps=problematic_apps,
            )

        if modified_files:
            message += "\n\n" + m18n.n(
                "migration_0021_modified_files", manually_modified_files=modified_files
            )

        return message

    def patch_apt_sources_list(self):

        sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
        sources_list.append("/etc/apt/sources.list")

        # This :
        # - replace single 'buster' occurence by 'bulleye'
        # - comments lines containing "backports"
        # - replace 'buster/updates' by 'bullseye/updates' (or same with -)
        # Special note about the security suite:
        # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive
        for f in sources_list:
            command = (
                f"sed -i {f} "
                "-e 's@ buster @ bullseye @g' "
                "-e '/backports/ s@^#*@#@' "
                "-e 's@ buster/updates @ bullseye-security @g' "
                "-e 's@ buster-@ bullseye-@g' "
            )
            os.system(command)

    def get_apps_equivs_packages(self):

        command = (
            "dpkg --get-selections"
            " | grep -v deinstall"
            " | awk '{print $1}'"
            " | { grep 'ynh-deps$' || true; }"
        )

        output = check_output(command)

        return output.split("\n") if output else []

    def hold(self, packages):
        for package in packages:
            os.system("apt-mark hold {}".format(package))

    def unhold(self, packages):
        for package in packages:
            os.system("apt-mark unhold {}".format(package))

    def apt_install(self, cmd):
        def is_relevant(line):
            return "Reading database ..." not in line.rstrip()

        callbacks = (
            lambda l: logger.info("+ " + l.rstrip() + "\r")
            if is_relevant(l)
            else logger.debug(l.rstrip() + "\r"),
            lambda l: logger.warning(l.rstrip()),
        )

        cmd = (
            "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes "
            + cmd
        )

        logger.debug("Running: %s" % cmd)

        call_async_output(cmd, callbacks, shell=True)

    def patch_yunohost_conflicts(self):
        #
        # This is a super dirty hack to remove the conflicts from yunohost's debian/control file
        # Those conflicts are there to prevent mistakenly upgrading critical packages
        # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly
        # using backports etc.
        #
        # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status
        #

        # We only patch the conflict if we're on yunohost 4.x
        if self.yunohost_major_version() != N_CURRENT_YUNOHOST:
            return

        conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip()
        if conflicts:
            # We want to keep conflicting with apache/bind9 tho
            new_conflicts = "Conflicts: apache2, bind9"

            command = f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'"
            logger.debug(f"Running: {command}")
            os.system(command)
<commit_msg>Aaaand typo T_T<commit_after>import glob
import os

from moulinette import m18n
from yunohost.utils.error import YunohostError
from moulinette.utils.log import getActionLogger
from moulinette.utils.process import check_output, call_async_output
from moulinette.utils.filesystem import read_file, rm

from yunohost.tools import Migration, tools_update, tools_upgrade
from yunohost.app import unstable_apps
from yunohost.regenconf import manually_modified_files, _force_clear_hashes
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.utils.packages import (
    get_ynh_package_version,
    _list_upgradable_apt_packages,
)
from yunohost.service import _get_services, _save_services

logger = getActionLogger("yunohost.migration")

N_CURRENT_DEBIAN = 10
N_CURRENT_YUNOHOST = 4

N_NEXT_DEBAN = 11
N_NEXT_YUNOHOST = 11

class MyMigration(Migration):

    "Upgrade the system to Debian Bullseye and Yunohost 11.x"

    mode = "manual"

    def run(self):

        self.check_assertions()

        logger.info(m18n.n("migration_0021_start"))

        #
        # Add new apt .deb signing key
        #

        new_apt_key = "https://forge.yunohost.org/yunohost_bullseye.asc"
        check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -")

        #
        # Patch sources.list
        #
        logger.info(m18n.n("migration_0021_patching_sources_list"))
        self.patch_apt_sources_list()
        tools_update(target="system")

        # Tell libc6 it's okay to restart system stuff during the upgrade
        os.system(
            "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections"
        )

        # Don't send an email to root about the postgresql migration. It should be handled automatically after.
        os.system(
            "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections"
        )

        #
        # Patch yunohost conflicts
        #
        logger.info(m18n.n("migration_0021_patch_yunohost_conflicts"))

        self.patch_yunohost_conflicts()

        #
        # Specific tweaking to get rid of custom my.cnf and use debian's default one
        # (my.cnf is actually a symlink to mariadb.cnf)
        #

        _force_clear_hashes(["/etc/mysql/my.cnf"])
        rm("/etc/mysql/mariadb.cnf", force=True)
        rm("/etc/mysql/my.cnf", force=True)
        self.apt_install("mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'")

        #
        # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl
        #
        if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"):
            os.system("mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl")
            rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True)

        #
        # /home/yunohost.conf -> /var/cache/yunohost/regenconf
        #
        if os.path.exists("/home/yunohost.conf"):
            os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf")
            rm("/home/yunohost.conf", recursive=True, force=True)

        #
        # Main upgrade
        #
        logger.info(m18n.n("migration_0021_main_upgrade"))

        apps_packages = self.get_apps_equivs_packages()
        self.hold(apps_packages)
        tools_upgrade(target="system", allow_yunohost_upgrade=False)

        if self.debian_major_version() == N_CURRENT_DEBIAN:
            raise YunohostError("migration_0021_still_on_buster_after_main_upgrade")

        # Clean the mess
        logger.info(m18n.n("migration_0021_cleaning_up"))
        os.system("apt autoremove --assume-yes")
        os.system("apt clean --assume-yes")

        # Force add sury if it's not there yet
        # This is to solve some weird issue with php-common breaking php7.3-common,
        # hence breaking many php7.3-deps
        # hence triggering some dependency conflict (or foobar-ynh-deps uninstall)
        # Adding it there shouldnt be a big deal - Yunohost 11.x does add it
        # through its regen conf anyway.
        if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"):
            open("/etc/apt/sources.list.d/extra_php_version.list", "w").write("deb https://packages.sury.org/php/ bullseye main")
            os.system('wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"')

        os.system("apt update")

        # Force explicit install of php7.4-fpm to make sure it's ll be there
        # during 0022_php73_to_php74_pools migration
        self.apt_install("php7.4-fpm -o Dpkg::Options::='--force-confmiss'")

        # Remove legacy postgresql service record added by helpers,
        # will now be dynamically handled by the core in bullseye
        services = _get_services()
        if "postgresql" in services:
            del services["postgresql"]
            _save_services(services)

        #
        # Yunohost upgrade
        #
        logger.info(m18n.n("migration_0021_yunohost_upgrade"))
        self.unhold(apps_packages)
        tools_upgrade(target="system")

    def debian_major_version(self):
        # The python module "platform" and lsb_release are not reliable because
        # on some setup, they may still return Release=9 even after upgrading to
        # buster ... (Apparently this is related to OVH overriding some stuff
        # with /etc/lsb-release for instance -_-)
        # Instead, we rely on /etc/os-release which should be the raw info from
        # the distribution...
        return int(
            check_output(
                "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2"
            )
        )

    def yunohost_major_version(self):
        return int(get_ynh_package_version("yunohost")["version"].split(".")[0])

    def check_assertions(self):

        # Be on buster (10.x) and yunohost 4.x
        # NB : we do both check to cover situations where the upgrade crashed
        # in the middle and debian version could be > 9.x but yunohost package
        # would still be in 3.x...
        if (
            not self.debian_major_version() == N_CURRENT_DEBIAN
            and not self.yunohost_major_version() == N_CURRENT_YUNOHOST
        ):
            raise YunohostError("migration_0021_not_buster")

        # Have > 1 Go free space on /var/ ?
        if free_space_in_directory("/var/") / (1024 ** 3) < 1.0:
            raise YunohostError("migration_0021_not_enough_free_space")

        # Check system is up to date
        # (but we don't if 'bullseye' is already in the sources.list ...
        # which means maybe a previous upgrade crashed and we're re-running it)
        if " bullseye " not in read_file("/etc/apt/sources.list"):
            tools_update(target="system")
            upgradable_system_packages = list(_list_upgradable_apt_packages())
            if upgradable_system_packages:
                raise YunohostError("migration_0021_system_not_fully_up_to_date")

    @property
    def disclaimer(self):

        # Avoid having a super long disclaimer + uncessary check if we ain't
        # on buster / yunohost 4.x anymore
        # NB : we do both check to cover situations where the upgrade crashed
        # in the middle and debian version could be >= 10.x but yunohost package
        # would still be in 4.x...
        if (
            not self.debian_major_version() == N_CURRENT_DEBIAN
            and not self.yunohost_major_version() == N_CURRENT_YUNOHOST
        ):
            return None

        # Get list of problematic apps ? I.e. not official or community+working
        problematic_apps = unstable_apps()
        problematic_apps = "".join(["\n    - " + app for app in problematic_apps])

        # Manually modified files ? (c.f. yunohost service regen-conf)
        modified_files = manually_modified_files()
        modified_files = "".join(["\n    - " + f for f in modified_files])

        message = m18n.n("migration_0021_general_warning")

        # FIXME: re-enable this message with updated topic link once we release the migration as stable
        #message = (
        #    "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/12195\n\n"
        #    + message
        #)

        if problematic_apps:
            message += "\n\n" + m18n.n(
                "migration_0021_problematic_apps_warning",
                problematic_apps=problematic_apps,
            )

        if modified_files:
            message += "\n\n" + m18n.n(
                "migration_0021_modified_files", manually_modified_files=modified_files
            )

        return message

    def patch_apt_sources_list(self):

        sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
        sources_list.append("/etc/apt/sources.list")

        # This :
        # - replace single 'buster' occurence by 'bulleye'
        # - comments lines containing "backports"
        # - replace 'buster/updates' by 'bullseye/updates' (or same with -)
        # Special note about the security suite:
        # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive
        for f in sources_list:
            command = (
                f"sed -i {f} "
                "-e 's@ buster @ bullseye @g' "
                "-e '/backports/ s@^#*@#@' "
                "-e 's@ buster/updates @ bullseye-security @g' "
                "-e 's@ buster-@ bullseye-@g' "
            )
            os.system(command)

    def get_apps_equivs_packages(self):

        command = (
            "dpkg --get-selections"
            " | grep -v deinstall"
            " | awk '{print $1}'"
            " | { grep 'ynh-deps$' || true; }"
        )

        output = check_output(command)

        return output.split("\n") if output else []

    def hold(self, packages):
        for package in packages:
            os.system("apt-mark hold {}".format(package))

    def unhold(self, packages):
        for package in packages:
            os.system("apt-mark unhold {}".format(package))

    def apt_install(self, cmd):
        def is_relevant(line):
            return "Reading database ..." not in line.rstrip()

        callbacks = (
            lambda l: logger.info("+ " + l.rstrip() + "\r")
            if is_relevant(l)
            else logger.debug(l.rstrip() + "\r"),
            lambda l: logger.warning(l.rstrip()),
        )

        cmd = (
            "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes "
            + cmd
        )

        logger.debug("Running: %s" % cmd)

        call_async_output(cmd, callbacks, shell=True)

    def patch_yunohost_conflicts(self):
        #
        # This is a super dirty hack to remove the conflicts from yunohost's debian/control file
        # Those conflicts are there to prevent mistakenly upgrading critical packages
        # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly
        # using backports etc.
        #
        # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status
        #

        # We only patch the conflict if we're on yunohost 4.x
        if self.yunohost_major_version() != N_CURRENT_YUNOHOST:
            return

        conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip()
        if conflicts:
            # We want to keep conflicting with apache/bind9 tho
            new_conflicts = "Conflicts: apache2, bind9"

            command = f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'"
            logger.debug(f"Running: {command}")
            os.system(command)
<|endoftext|>


<commit_before>"""
    word2vec implementation via Gensim, training over our LiveJournal Corpus
    Following guideline of code from the forum at: https://groups.google.com/forum/#!topic/gensim/MJWrDw_IvXw

    Currently trying to train over 1131 separate text files.
    If this is too slow and/or memory inefficient, we will probably just concatentate all text files into one
    and rerun.
"""

import logging
import os.path
import sys
import multiprocessing
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

if __name__ == '__main__':

    program = os.path.basename(sys.argv[0])
    logger = logging.getLogger(program)
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    sentDirectory = '/var/local/destress/text_sent_ids/'
    outputModel = '/var/local/destress/word2vecLJ.model'

    fileName = 'sents_1'
    sentences = LineSentence(sentDirectory+fileName)
    model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count())
    model.save(outputModel)

    #This feels very inefficient... but AFAIK, gensim requires you to load/save each time for multiple files

    for i in range(2, 1132):
        #load the previous model
        model = Word2Vec.load(outputModel)
        fileName = 'sents_' + str(i)
        sentences = LineSentence(sentDirectory+fileName)
        model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count())
        model.save(outputModel)

"""
    When we finish training w/ negative sampling (after we pick out the "bad queries")
    we can save a LOT of Ram with the following:
    model.init_sims(replace=True)

"""
<commit_msg>Renamed input name<commit_after>"""
    word2vec implementation via Gensim, training over our LiveJournal Corpus
    Following guideline of code from the forum at: https://groups.google.com/forum/#!topic/gensim/MJWrDw_IvXw

    Currently trying to train over 1131 separate text files.
    If this is too slow and/or memory inefficient, we will probably just concatentate all text files into one
    and rerun.
"""

import logging
import os.path
import sys
import multiprocessing
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

if __name__ == '__main__':

    program = os.path.basename(sys.argv[0])
    logger = logging.getLogger(program)
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
    logging.root.setLevel(level=logging.INFO)
    logger.info("running %s" % ' '.join(sys.argv))

    sentDirectory = '/var/local/destress/text_sent_ids/'
    outputModel = '/var/local/destress/word2vecLJ.model'

    fileName = 'sents_1'
    sentences = LineSentence(sentDirectory+fileName)
    model = Word2Vec(LineSentence(sentences), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count())
    model.save(outputModel)

    #This feels very inefficient... but AFAIK, gensim requires you to load/save each time for multiple files

    for i in range(2, 1132):
        #load the previous model
        model = Word2Vec.load(outputModel)
        fileName = 'sents_' + str(i)
        sentences = LineSentence(sentDirectory+fileName)
        model = Word2Vec(LineSentence(sentences), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count())
        model.save(outputModel)

"""
    When we finish training w/ negative sampling (after we pick out the "bad queries")
    we can save a LOT of Ram with the following:
    model.init_sims(replace=True)

"""
<|endoftext|>


<commit_before>import arcpy
import os
import pandas
from scripts import wqt_timestamp_match

class Toolbox(object):
	def __init__(self):
		"""Define the toolbox (the name of the toolbox is the name of the .pyt file)."""
		self.label = "ArcWQ"
		self.alias = ""

		# List of tool classes associated with this toolbox
		self.tools = [checkmatch, wqt2shp, gain2shp]

class JoinTimestamp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join on Timestamp"
		self.description = ""
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		csvs = arcpy.Parameter(
			displayName="Transect Water Quality Data",
			name="csv_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEFeatureClass",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Joined Output",
			name="Output",
			datatype="DEFeatureClass",
			direction="Output"
		)

		params = [csvs, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		param1 = parameters[0].value.exportToString()
		wq_transect_list = param1.split(";")

		pts = arcpy.GetParameterAsText(1)
		arcpy.AddMessage(pts)

		out = arcpy.GetParameterAsText(2)

		# list of water quality files from parameter
		#wq = wqt_timestamp_match.wq_append_fromlist(wq_transect_list)
		wq = wq_transect_list[0] # TODO

		# run wq_join_match
		wqt_timestamp_match.main(wq, pts, out)

		return

class checkmatch(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Percent Match - Water Quality data with Transect"
		self.description = "Reports the percent match for multiple water quality dataset with transect shapefile"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		csvs = arcpy.Parameter(
			displayName="Transect Water Quality Data",
			name="wqt_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		params = [csvs, bc]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		param1 = parameters[0].valueAsText
		wq_transect_list = param1.split(";")

		pts = str(parameters[1].valueAsText)
		shp_df = wqt_timestamp_match.wqtshp2pd(pts)

		for wq in wq_transect_list:
			basename = os.path.basename(wq)
			wq_df = wqt_timestamp_match.wq_from_file(wq)
			ts_join = wqt_timestamp_match.JoinByTimeStamp(wq_df, shp_df)
			ts_results = wqt_timestamp_match.splitunmatched(ts_join)
			percent = wqt_timestamp_match.JoinMatchPercent(wq_df, ts_results[0])

			arcpy.AddMessage("{} has a {} % match with the transect. ".format(basename, percent))

		return

class wqt2shp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join WQT to SHP"
		self.description = "Matches Water Quality data with Transect using timestamps"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		wq_data = arcpy.Parameter(
			displayName="Transect Water Quality Data (wqt)",
			name="wqt_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Output Feature Class",
			name="out_file",
			datatype="DEShapefile",
			direction="Output"
		)

		params = [wq_data, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""

		# get the parameters
		param1 = parameters[0].valueAsText
		wq_transect_list = param1.split(";") # the multi input needs to be split
		gps_pts = str(parameters[1].valueAsText)
		output_feature = parameters[2].valueAsText

		# see wqt_timestamp_match for functions
		wqt_timestamp_match.main(wq_transect_list, gps_pts, output_feature)

		return

class gain2shp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join Gain profile average to SHP"
		self.description = "Matches vertical Water Quality data with Transect using site names"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		wqp = arcpy.Parameter(
			displayName="Vertical Profile file (wqp)",
			name="wqp_files",
			datatype="GPValueTable",
			parameterType='Required',
			multiValue=True,
			direction="Input"
		)

		wqp.columns = [['DEFile', 'WQP'], ['GPString', 'Site ID'], ['GPString', 'Gain Type']]
		wqp.filters[1].type = 'ValueList'
		wqp.filters[1].list = ['BK1', 'CA1', 'CA3', 'CC1', 'LNCA', 'UL1']
		wqp.filters[2].type = 'ValueList'
		wqp.filters[2].list = ['g0', 'g1', 'g10', 'g100']

		# shapefile for the stationary GPS points
		bc = arcpy.Parameter(
			displayName="WQP/Zoop/Chl Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Output Feature Class",
			name="out_file",
			datatype="DEShapefile",
			direction="Output"
		)

		params = [wqp, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		arcpy.AddMessage(parameters[0])
		# get the parameters
		param = parameters[0].valueAsText
		file_params = param.split(";") # the multi input needs to be split
		wqp = []
		for f in file_params:
			arcpy.AddMessage(f)
			# MAKE SURE FILE does not have spaces!!!!
			row = f.split(" ") # split using single space
			arcpy.AddMessage(row)
			wqp.append(row)

		arcpy.AddMessage(wqp)

		# gps_pts = str(parameters[1].valueAsText)
		# output_feature = parameters[2].valueAsText
		#
		# # see wqt_timestamp_match for functions
		# wqt_timestamp_match.main(wq_transect_list, gps_pts, output_feature)
		pass

		return<commit_msg>added in gain processing<commit_after>import arcpy
import os
import pandas
from scripts import wqt_timestamp_match
from scripts import wq_gain

class Toolbox(object):
	def __init__(self):
		"""Define the toolbox (the name of the toolbox is the name of the .pyt file)."""
		self.label = "ArcWQ"
		self.alias = ""

		# List of tool classes associated with this toolbox
		self.tools = [checkmatch, wqt2shp, gain2shp]

class JoinTimestamp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join on Timestamp"
		self.description = ""
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		csvs = arcpy.Parameter(
			displayName="Transect Water Quality Data",
			name="csv_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEFeatureClass",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Joined Output",
			name="Output",
			datatype="DEFeatureClass",
			direction="Output"
		)

		params = [csvs, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		param1 = parameters[0].value.exportToString()
		wq_transect_list = param1.split(";")

		pts = arcpy.GetParameterAsText(1)
		arcpy.AddMessage(pts)

		out = arcpy.GetParameterAsText(2)

		# list of water quality files from parameter
		#wq = wqt_timestamp_match.wq_append_fromlist(wq_transect_list)
		wq = wq_transect_list[0] # TODO

		# run wq_join_match
		wqt_timestamp_match.main(wq, pts, out)

		return

class checkmatch(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Percent Match - Water Quality data with Transect"
		self.description = "Reports the percent match for multiple water quality dataset with transect shapefile"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		csvs = arcpy.Parameter(
			displayName="Transect Water Quality Data",
			name="wqt_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		params = [csvs, bc]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		param1 = parameters[0].valueAsText
		wq_transect_list = param1.split(";")

		pts = str(parameters[1].valueAsText)
		shp_df = wqt_timestamp_match.wqtshp2pd(pts)

		for wq in wq_transect_list:
			basename = os.path.basename(wq)
			wq_df = wqt_timestamp_match.wq_from_file(wq)
			ts_join = wqt_timestamp_match.JoinByTimeStamp(wq_df, shp_df)
			ts_results = wqt_timestamp_match.splitunmatched(ts_join)
			percent = wqt_timestamp_match.JoinMatchPercent(wq_df, ts_results[0])

			arcpy.AddMessage("{} has a {} % match with the transect. ".format(basename, percent))

		return

class wqt2shp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join WQT to SHP"
		self.description = "Matches Water Quality data with Transect using timestamps"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		wq_data = arcpy.Parameter(
			displayName="Transect Water Quality Data (wqt)",
			name="wqt_files",
			datatype="DEFile",
			multiValue=True,
			direction="Input"
		)

		# shapefile for the transects GPS breadcrumbs
		bc = arcpy.Parameter(
			displayName="Transect Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Output Feature Class",
			name="out_file",
			datatype="DEShapefile",
			direction="Output"
		)

		params = [wq_data, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""

		# get the parameters
		param1 = parameters[0].valueAsText
		wq_transect_list = param1.split(";") # the multi input needs to be split
		gps_pts = str(parameters[1].valueAsText)
		output_feature = parameters[2].valueAsText

		# see wqt_timestamp_match for functions
		wqt_timestamp_match.main(wq_transect_list, gps_pts, output_feature)

		return

class gain2shp(object):
	def __init__(self):
		"""Define the tool (tool name is the name of the class)."""
		self.label = "Join Gain profile average to SHP"
		self.description = "Matches vertical Water Quality data with Transect using site names"
		self.canRunInBackground = False

	def getParameterInfo(self):
		"""Define parameter definitions"""

		# parameter info for selecting multiple csv water quality files
		wqp = arcpy.Parameter(
			displayName="Vertical Profile file (wqp)",
			name="wqp_files",
			datatype="GPValueTable",
			parameterType='Required',
			multiValue=True,
			direction="Input"
		)

		wqp.columns = [['DEFile', 'WQP'], ['GPString', 'Site ID'], ['GPString', 'Gain Type']]
		wqp.filters[1].type = 'ValueList'
		wqp.filters[1].list = ['BK1', 'CA1', 'CA3', 'CC1', 'LNCA', 'UL1']
		wqp.filters[2].type = 'ValueList'
		wqp.filters[2].list = ['g0', 'g1', 'g10', 'g100']

		# shapefile for the stationary GPS points
		bc = arcpy.Parameter(
			displayName="WQP/Zoop/Chl Shapefile",
			name="shp_file",
			datatype="DEShapefile",
			direction="Input"
		)

		out = arcpy.Parameter(
			displayName="Output Feature Class",
			name="out_file",
			datatype="DEShapefile",
			direction="Output"
		)

		params = [wqp, bc, out]
		return params

	def isLicensed(self):
		"""Set whether tool is licensed to execute."""
		return True

	def updateParameters(self, parameters):
		"""Modify the values and properties of parameters before internal
		validation is performed.  This method is called whenever a parameter
		has been changed."""
		return

	def updateMessages(self, parameters):
		"""Modify the messages created by internal validation for each tool
		parameter.  This method is called after internal validation."""
		return

	def execute(self, parameters, messages):
		"""The source code of the tool."""
		# get the parameters
		param = parameters[0].valueAsText
		file_params = param.split(";") # the multi input needs to be split
		wqps = []
		for f in file_params:
			# MAKE SURE FILE does not have spaces!!!!
			row = f.split(" ") # split using single space
			wqps.append(row)

		arcpy.AddMessage(wqps)

		gps_pts = str(parameters[1].valueAsText)
		output_feature = parameters[2].valueAsText

		master_wq_df = pandas.DataFrame()

		for wq in wqps:
			wq_gain_file = wq[0]
			site_id = wq[1]
			gain_setting = wq[2]

			join_df = wq_gain.main(wq_gain_file, site_id, gain_setting, gps_pts)

			# append to master wq
			master_wq_df = master_wq_df.append(join_df)

		arcpy.AddMessage(master_wq_df.head())

		return<|endoftext|>


<commit_before>import os
import json
import requests

from django.http import JsonResponse
#TODO: Gjøre unpack som tupler i stedet
def hent_vaktliste():
    """
    cur_dir = os.path.dirname(os.path.realpath(__file__))
    vaktliste = os.path.join(cur_dir, "vaktliste.json")
    with open(vaktliste,"r") as f:
        return json.loads(f.read())
    """
    return requests.get("https://script.googleusercontent.com/macros/echo?user_content_key=gR05slZZQrkrumxUc8DJZEc81FUEXWJpVDu8OGmYc7Bd8STD9BEvHnNLn3Hqa93sZAkXhzOJJfVsxRBCis22hxj50ZyEv7V0m5_BxDlH2jW0nuo2oDemN9CCS2h10ox_1xSncGQajx_ryfhECjZEnBHVJ4Ip7UlCqkboOF3idyLswydE_Rh_IZ2xA43kME624RrB2b1T6_LZIUQtyudpTtsAUXIaJqQ5&lib=MvQgEbo5GAfi_xTmCXLhSAK0T_1fexhuo").json()

def vakt_filter(dager,tider):
    filter_data = {}
    filter_times = []
    filter_days = [d.title() for d in dager if d!='']
    for tid in tider:
        #HERE BE DRAGONS
        if tid!='':
            time_slots = ["10:15 - 12:07","12:07 - 14:07", "14:07 - 16:07", "16:07 - 18:00"]
            h,m = map(int,tid.split(":"))
            timeslot = ""
            #WHY ARE YOU STILL HERE
            if h>=18:
                filter_times.append("16:07 - 18:00")
            elif h<10:
                filter_times.append("10:15 - 12:07")
            else:
                for t in range(10,18,2):
                    if h in range(t,t+2):
                        #AAAAAAAAAAAAAAAAAAAAH
                        if h==t and m<7:
                            filter_times.append(time_slots[max(0,(t-10)//2-1)])
                        else:
                            filter_times.append(time_slots[(t-10)//2])
        #THE DRAGONS ARE GONE
    vakt_data = hent_vaktliste()
    for day in vakt_data:
        if len(filter_days)==0 or day in filter_days: # Dagen slipper gjennom filteret
            if len(filter_times)==0: # what is time
                filter_data[day] = vakt_data[day]
            else:
                vakter = {}
                for time_slot in vakt_data[day]:
                    if time_slot in filter_times: # Vakttiden slipper gjennom filteret
                        vakter[time_slot] = vakt_data[day][time_slot]
                filter_data[day] = vakter

    return filter_data

def finn_person(person,full=True):
    filter_data = {}
    vakt_data = hent_vaktliste()
    for day in vakt_data:
        for time in vakt_data[day]:
            for hacker in [p.lower() for p in vakt_data[day][time]]:
                if person.lower() in hacker: #TODO: Gjøre dette til fuzzy finding
                    if full:
                        if day in filter_data:
                            filter_data[day][time]=vakt_data[day][time]
                        else:
                            filter_data[day] = {time: vakt_data[day][time]}
                    else:
                        if day in filter_data:
                            filter_data[day][time]=[hacker.title()]
                        else:
                            filter_data[day] = {time: [hacker.title()]}
                    break
    if len(filter_data)==0:
        return {}
    else:
        return filter_data

def vakter(request):
    dager = request.GET.get('dag','')
    tider = request.GET.get('tid','')
    vakt_data = vakt_filter(dager.split(","),tider.split(","))
    return JsonResponse(vakt_data)

def personsok(request):
    person = request.GET.get('person','')
    if person=='': person="FACK"
    full = request.GET.get('full','')
    if full in ['', 'True']:
        full = True
    elif full=="False":
        full = False
    vakt_data = finn_person(person,full=full)
    return JsonResponse(vakt_data)
<commit_msg>Flyttet splitten<commit_after>import os
import json
import requests

from django.http import JsonResponse
#TODO: Gjøre unpack som tupler i stedet
def hent_vaktliste():
    """
    cur_dir = os.path.dirname(os.path.realpath(__file__))
    vaktliste = os.path.join(cur_dir, "vaktliste.json")
    with open(vaktliste,"r") as f:
        return json.loads(f.read())
    """
    return requests.get("https://script.googleusercontent.com/macros/echo?user_content_key=gR05slZZQrkrumxUc8DJZEc81FUEXWJpVDu8OGmYc7Bd8STD9BEvHnNLn3Hqa93sZAkXhzOJJfVsxRBCis22hxj50ZyEv7V0m5_BxDlH2jW0nuo2oDemN9CCS2h10ox_1xSncGQajx_ryfhECjZEnBHVJ4Ip7UlCqkboOF3idyLswydE_Rh_IZ2xA43kME624RrB2b1T6_LZIUQtyudpTtsAUXIaJqQ5&lib=MvQgEbo5GAfi_xTmCXLhSAK0T_1fexhuo").json()

def vakt_filter(dager,tider):
    filter_data = {}
    filter_times = []
    filter_days = [d.title() for d in dager.split(",") if d!='']
    for tid in tider.split(","):
        #HERE BE DRAGONS
        if tid!='':
            time_slots = ["10:15 - 12:07","12:07 - 14:07", "14:07 - 16:07", "16:07 - 18:00"]
            h,m = map(int,tid.split(":"))
            timeslot = ""
            #WHY ARE YOU STILL HERE
            if h>=18:
                filter_times.append("16:07 - 18:00")
            elif h<10:
                filter_times.append("10:15 - 12:07")
            else:
                for t in range(10,18,2):
                    if h in range(t,t+2):
                        #AAAAAAAAAAAAAAAAAAAAH
                        if h==t and m<7:
                            filter_times.append(time_slots[max(0,(t-10)//2-1)])
                        else:
                            filter_times.append(time_slots[(t-10)//2])
        #THE DRAGONS ARE GONE
    vakt_data = hent_vaktliste()
    for day in vakt_data:
        if len(filter_days)==0 or day in filter_days: # Dagen slipper gjennom filteret
            if len(filter_times)==0: # what is time
                filter_data[day] = vakt_data[day]
            else:
                vakter = {}
                for time_slot in vakt_data[day]:
                    if time_slot in filter_times: # Vakttiden slipper gjennom filteret
                        vakter[time_slot] = vakt_data[day][time_slot]
                filter_data[day] = vakter

    return filter_data

def finn_person(persons,full=True):
    filter_data = {}
    vakt_data = hent_vaktliste()
    for person in persons.split(","):
        for day in vakt_data:
            for time in vakt_data[day]:
                for hacker in [p.lower() for p in vakt_data[day][time]]:
                    if person.lower() in hacker: #TODO: Gjøre dette til fuzzy finding
                        if full:
                            if day in filter_data:
                                filter_data[day][time]=vakt_data[day][time]
                            else:
                                filter_data[day] = {time: vakt_data[day][time]}
                        else:
                            if day in filter_data:
                                filter_data[day][time]=[hacker.title()]
                            else:
                                filter_data[day] = {time: [hacker.title()]}
                        break
    if len(filter_data)==0:
        return {}
    else:
        return filter_data

def vakter(request):
    dager = request.GET.get('dag','')
    tider = request.GET.get('tid','')
    vakt_data = vakt_filter(dager,tider)
    return JsonResponse(vakt_data)

def personsok(request):
    person = request.GET.get('person','FACK')
    full = request.GET.get('full','')
    if full in ['', 'True']:
        full = True
    elif full=="False":
        full = False
    vakt_data = finn_person(person,full=full)
    return JsonResponse(vakt_data)
<|endoftext|>