<commit_before># Copyright Hugh Perkins 2015 hughperkins at gmail
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

#from distutils.core import setup
import os
import os.path
import sysconfig
import sys
import glob
import platform
from setuptools import setup
#from distutils.extension import Extension
from setuptools import Extension
import distutils.dir_util
import distutils.file_util

cython_present = False
try:
    from Cython.Build import cythonize
    cython_present = True
except ImportError:
    pass

pypandoc_present = False
try:
    import pypandoc
    pypandoc_present = True
except ImportError:
    pass

#print ( sys.argv )
# if any of sys.argv is bdist or sdist or bdist_egg, then lets copy everything to
# a subfolder of us called 'mysrc', since '..' paths dont work well..
# otherwise, let's just assume this folder already contains our source :-)
docopy = False
for arg in sys.argv:
    if arg in ('sdist','bdist','bdist_egg','build_ext'):
        docopy = True

srcdirs = ['lua', 'activate','batch','clmath','conv','dropout','fc','forcebackprop',
    'input','layer','loaders','loss','net','netdef','normalize','patches',
    'pooling','trainers','util','weights', 'qlearning' ]

if docopy:
    if not os.path.isdir('mysrc'):
        os.makedirs('mysrc')
    if not os.path.isdir('mysrc/util'):
        os.makedirs('mysrc/util')
    if not os.path.isdir('mysrc/templates'):
        os.makedirs('mysrc/templates')
    if not os.path.isdir('mysrc/lua'):
        os.makedirs('mysrc/lua')
    for thisdir in ['../src','../EasyCL',
            '../EasyCL/thirdparty/clew/src']: # copy everything..
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/' + thisfile )
    for thisdir in ['../EasyCL/util']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/util/' + thisfile )
    for thisdir in ['../EasyCL/thirdparty/lua-5.1.5/src']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/lua/' + thisfile )
    distutils.file_util.copy_file('../EasyCL/thirdparty/lua-5.1.5/files.txt', 'mysrc/lua/files.txt')
    for thisdir in ['../EasyCL/templates']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/templates/' + thisfile )
    distutils.file_util.copy_file( '../jenkins/version.txt', 'version.txt' )
    for srcdir in srcdirs:
        if srcdir == 'lua':
            continue
        if not os.path.isdir('mysrc/' + srcdir):
            os.makedirs('mysrc/' + srcdir)
        for thisfile in os.listdir('../src/' + srcdir):
            thisfilepath = '../src/' + srcdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/' + srcdir + '/' + thisfile )

#        distutils.dir_util.copy_tree( thisdir, 'mysrc' )

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

def get_so_suffix():
    if sysconfig.get_config_var('SOABI') != None:
        return "." + sysconfig.get_config_var('SOABI')
    return ""

if pypandoc_present:
    pypandoc.convert('README.md', 'rst', outputfile = 'README.rst' )

def my_cythonize(extensions, **_ignore):
    #newextensions = []
    for extension in extensions:
        print(extension.sources)
        should_cythonize = False
        sources = []
        for sfile in extension.sources:
            path, ext = os.path.splitext(sfile)
            if ext in ('.pyx', '.py'):
                should_cythonize = True
                if not cython_present:
                   # if extension.language == 'c++':
                        ext = '.cpp'
                    #else:
                     #   ext = '.c'
            if sfile == 'PyDeepCL.c':
                ext = '.cpp' # hack for now... not sure how to fix this cleanly
                             # yet
            sfile = path + ext
            if sfile.startswith('..'):
                # use mysrc instead
                basename = os.path.basename(sfile)
                sfile = 'mysrc/' + basename
            sources.append(sfile)
        #print(should_cythonize)
        if should_cythonize and cython_present:
            print('cythonizing...')
            cythonize(extension)
        extension.sources[:] = sources
        #newextensions.append( extension )
    return extensions

def no_cythonize(extensions, **_ignore):
    for extension in extensions:
        sources = []
        for sfile in extension.sources:
            path, ext = os.path.splitext(sfile)
            #print('path,ext',path,ext)
            if ext in ('.pyx', '.py'):
                #if extension.language == 'c++':
                    ext = '.cpp'
                #else:
                #    ext = '.c'
            if sfile == 'PyDeepCL.c':
                ext = '.cpp' # hack for now... not sure how to fix this cleanly
                             # yet
            sfile = path + ext
            if sfile.startswith('..'):
                # use mysrc instead
                basename = os.path.basename(sfile)
                sfile = 'mysrc/' + basename
            sources.append(sfile)
            print('appending source ', sfile )
        extension.sources[:] = sources
    return extensions

# from http://stackoverflow.com/questions/14320220/testing-python-c-libraries-get-build-path
def distutils_dir_name(dname):
    """Returns the name of a distutils build directory"""
    f = "{dirname}.{platform}-{version[0]}.{version[1]}"
    return f.format(dirname=dname,
                    platform=sysconfig.get_platform(),
                    version=sys.version_info)

def lib_build_dir():
    return os.path.join('build', distutils_dir_name('lib'))

deepcl_sources = []
for srcdir in srcdirs:
    filespath = 'mysrc/' + srcdir + '/files.txt'
    fileslist = []
    with open( filespath, 'r' ) as f:
        lines = f.readlines()
        for line in lines:
            if line.strip() != "":
                fileslist.append( 'mysrc/' + srcdir + '/' + line.strip() )
#    print('fileslist: ', fileslist)
    deepcl_sources = deepcl_sources + fileslist
print('deeplcl_sources', deepcl_sources)
#for source in deepcl_sources_all:
#    deepcl_sources.append(source)

easyclsources = list(map( lambda name : 'mysrc/' + name, [
        'EasyCL.cpp',
        'deviceinfo_helper.cpp', 'platforminfo_helper.cpp',
         'templates/LuaTemplater.cpp',
        'util/easycl_stringhelper.cpp', 'templates/TemplatedKernel.cpp',
#        'EasyCL/speedtemplates/SpeedTemplates.cpp',
        'CLWrapper.cpp',
        'CLKernel.cpp', 'clew.c' ] ))
print(easyclsources)
print(isinstance( easyclsources, list) )

compile_options = []
osfamily = platform.uname()[0]
if osfamily == 'Windows':
   compile_options.append('/EHsc')
elif osfamily == 'Linux':
   compile_options.append('-std=c++0x')
   compile_options.append('-g')
else:
   pass
   # put other options etc here if necessary

runtime_library_dirs = []
libraries = []
if osfamily == 'Linux':
    runtime_library_dirs= ['.']

if osfamily == 'Windows':
    libraries = ['winmm']

if cython_present:
    my_cythonize = cythonize
else:
    my_cythonize = no_cythonize

#libraries = [
#    ("EasyCL", {
#        'sources': easyclsources + ['dummy_easycl.cpp'],
#        'include_dirs': ['DeepCL/EasyCL'],
#        'extra_compile_args': compile_options,
##        define_macros = [('EasyCL_EXPORTS',1)],
##        libraries = []
##        language='c++'
#        }
#    )
#]

ext_modules = [
#    Extension("_EasyCL",
#        sources = easyclsources + ['dummy_easycl.cpp'],
#        include_dirs = ['DeepCL/EasyCL'],
#        extra_compile_args=compile_options,
#        define_macros = [('EasyCL_EXPORTS',1),('MS_WIN32',1)],
##        libraries = []
##        language='c++'
#    )
#    Extension("libDeepCL",
#        list(map( lambda name : 'DeepCL/src/' + name, deepcl_sources)), # +
##            glob.glob('DeepCL/src/*.h'),
#        include_dirs = ['DeepCL/src','DeepCL/EasyCL'],
#        extra_compile_args = compile_options,
#        library_dirs = [ lib_build_dir() ],
#        libraries = [ "EasyCL" + get_so_suffix() ],
#        define_macros = [('DeepCL_EXPORTS',1)],
#        runtime_library_dirs=runtime_library_dirs
##        language='c++'
#    ),
    Extension("PyDeepCL",
              sources=["PyDeepCL.pyx", 'CyWrappers.cpp']
                + easyclsources
                + deepcl_sources,
#                glob.glob('DeepCL/EasyCL/*.h'),
              include_dirs = ['mysrc', 'mysrc/lua'],
              libraries= libraries,
              extra_compile_args=compile_options,
        define_macros = [('DeepCL_EXPORTS',1),('EasyCL_EXPORTS',1)],
#              extra_objects=['cDeepCL.pxd'],
#              library_dirs = [lib_build_dir()],
              runtime_library_dirs=runtime_library_dirs,
              language="c++"
    )
]

def read_if_exists(filename):
    filepath = os.path.join(os.path.dirname(__file__), filename)
    if os.path.isfile(filepath):
        return open(filepath).read()
    else:
        ""

version = read_if_exists('version.txt').strip().replace('v', '')
print('version: ', version )

setup(
  name = 'DeepCL',
  version = version,
#  version = "3.4.0rc1",  # synchronize to deepcl main version
  author = "Hugh Perkins",
  author_email = "hughperkins@gmail.com",
  description = 'python wrapper for DeepCL deep convolutional neural network library for OpenCL',
  license = 'MPL',
  url = 'https://github.com/hughperkins/DeepCL',
  long_description = read('README.rst'),
  classifiers = [
    'Development Status :: 4 - Beta',
    'Topic :: Scientific/Engineering :: Artificial Intelligence',
    'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
  ],
  install_requires = [],
  # install_requires = [],
  tests_require = ['nose>=1.3.4','Cython>=0.22','cogapp>=2.4','future>=0.14.3'],
  scripts = ['test_deepcl.py','test_lowlevel.py'],
 # modules = libraries,
#  lib raries = libraries,
  ext_modules = my_cythonize( ext_modules),
)

<commit_msg>fix python build<commit_after># Copyright Hugh Perkins 2015 hughperkins at gmail
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

#from distutils.core import setup
import os
import os.path
import sysconfig
import sys
import glob
import platform
from setuptools import setup
#from distutils.extension import Extension
from setuptools import Extension
import distutils.dir_util
import distutils.file_util

cython_present = False
try:
    from Cython.Build import cythonize
    cython_present = True
except ImportError:
    pass

pypandoc_present = False
try:
    import pypandoc
    pypandoc_present = True
except ImportError:
    pass

#print ( sys.argv )
# if any of sys.argv is bdist or sdist or bdist_egg, then lets copy everything to
# a subfolder of us called 'mysrc', since '..' paths dont work well..
# otherwise, let's just assume this folder already contains our source :-)
docopy = False
for arg in sys.argv:
    if arg in ('sdist','bdist','bdist_egg','build_ext'):
        docopy = True

srcdirs = ['lua', 'activate','batch','clmath','conv','dropout','fc','forcebackprop',
    'input','layer','loaders','loss','net','netdef','normalize','patches',
    'pooling','trainers','util','weights', 'qlearning' ]

if docopy:
    if not os.path.isdir('mysrc'):
        os.makedirs('mysrc')
    if not os.path.isdir('mysrc/util'):
        os.makedirs('mysrc/util')
    if not os.path.isdir('mysrc/templates'):
        os.makedirs('mysrc/templates')
    if not os.path.isdir('mysrc/lua'):
        os.makedirs('mysrc/lua')
    for thisdir in ['../src','../EasyCL',
            '../EasyCL/thirdparty/clew/src',
            '../EasyCL/thirdparty/clew/include']: # copy everything..
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/' + thisfile )
    for thisdir in ['../EasyCL/util']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/util/' + thisfile )
    for thisdir in ['../EasyCL/thirdparty/lua-5.1.5/src']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/lua/' + thisfile )
    distutils.file_util.copy_file('../EasyCL/thirdparty/lua-5.1.5/files.txt', 'mysrc/lua/files.txt')
    for thisdir in ['../EasyCL/templates']:
        for thisfile in os.listdir(thisdir):
            #print(thisfile)
            thisfilepath = thisdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/templates/' + thisfile )
    distutils.file_util.copy_file( '../jenkins/version.txt', 'version.txt' )
    for srcdir in srcdirs:
        if srcdir == 'lua':
            continue
        if not os.path.isdir('mysrc/' + srcdir):
            os.makedirs('mysrc/' + srcdir)
        for thisfile in os.listdir('../src/' + srcdir):
            thisfilepath = '../src/' + srcdir +'/' + thisfile
            if os.path.isfile(thisfilepath):
                distutils.file_util.copy_file( thisfilepath, 'mysrc/' + srcdir + '/' + thisfile )

#        distutils.dir_util.copy_tree( thisdir, 'mysrc' )

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

def get_so_suffix():
    if sysconfig.get_config_var('SOABI') != None:
        return "." + sysconfig.get_config_var('SOABI')
    return ""

if pypandoc_present:
    pypandoc.convert('README.md', 'rst', outputfile = 'README.rst' )

def my_cythonize(extensions, **_ignore):
    #newextensions = []
    for extension in extensions:
        print(extension.sources)
        should_cythonize = False
        sources = []
        for sfile in extension.sources:
            path, ext = os.path.splitext(sfile)
            if ext in ('.pyx', '.py'):
                should_cythonize = True
                if not cython_present:
                   # if extension.language == 'c++':
                        ext = '.cpp'
                    #else:
                     #   ext = '.c'
            if sfile == 'PyDeepCL.c':
                ext = '.cpp' # hack for now... not sure how to fix this cleanly
                             # yet
            sfile = path + ext
            if sfile.startswith('..'):
                # use mysrc instead
                basename = os.path.basename(sfile)
                sfile = 'mysrc/' + basename
            sources.append(sfile)
        #print(should_cythonize)
        if should_cythonize and cython_present:
            print('cythonizing...')
            cythonize(extension)
        extension.sources[:] = sources
        #newextensions.append( extension )
    return extensions

def no_cythonize(extensions, **_ignore):
    for extension in extensions:
        sources = []
        for sfile in extension.sources:
            path, ext = os.path.splitext(sfile)
            #print('path,ext',path,ext)
            if ext in ('.pyx', '.py'):
                #if extension.language == 'c++':
                    ext = '.cpp'
                #else:
                #    ext = '.c'
            if sfile == 'PyDeepCL.c':
                ext = '.cpp' # hack for now... not sure how to fix this cleanly
                             # yet
            sfile = path + ext
            if sfile.startswith('..'):
                # use mysrc instead
                basename = os.path.basename(sfile)
                sfile = 'mysrc/' + basename
            sources.append(sfile)
            print('appending source ', sfile )
        extension.sources[:] = sources
    return extensions

# from http://stackoverflow.com/questions/14320220/testing-python-c-libraries-get-build-path
def distutils_dir_name(dname):
    """Returns the name of a distutils build directory"""
    f = "{dirname}.{platform}-{version[0]}.{version[1]}"
    return f.format(dirname=dname,
                    platform=sysconfig.get_platform(),
                    version=sys.version_info)

def lib_build_dir():
    return os.path.join('build', distutils_dir_name('lib'))

deepcl_sources = []
for srcdir in srcdirs:
    filespath = 'mysrc/' + srcdir + '/files.txt'
    fileslist = []
    with open( filespath, 'r' ) as f:
        lines = f.readlines()
        for line in lines:
            if line.strip() != "":
                fileslist.append( 'mysrc/' + srcdir + '/' + line.strip() )
#    print('fileslist: ', fileslist)
    deepcl_sources = deepcl_sources + fileslist
print('deeplcl_sources', deepcl_sources)
#for source in deepcl_sources_all:
#    deepcl_sources.append(source)

easyclsources = list(map( lambda name : 'mysrc/' + name, [
        'EasyCL.cpp',
        'deviceinfo_helper.cpp', 'platforminfo_helper.cpp',
         'templates/LuaTemplater.cpp',
        'util/easycl_stringhelper.cpp', 'templates/TemplatedKernel.cpp',
#        'EasyCL/speedtemplates/SpeedTemplates.cpp',
        'CLWrapper.cpp',
        'CLKernel.cpp', 'clew.c' ] ))
print(easyclsources)
print(isinstance( easyclsources, list) )

compile_options = []
osfamily = platform.uname()[0]
if osfamily == 'Windows':
   compile_options.append('/EHsc')
elif osfamily == 'Linux':
   compile_options.append('-std=c++0x')
   compile_options.append('-g')
else:
   pass
   # put other options etc here if necessary

runtime_library_dirs = []
libraries = []
if osfamily == 'Linux':
    runtime_library_dirs= ['.']

if osfamily == 'Windows':
    libraries = ['winmm']

if cython_present:
    my_cythonize = cythonize
else:
    my_cythonize = no_cythonize

#libraries = [
#    ("EasyCL", {
#        'sources': easyclsources + ['dummy_easycl.cpp'],
#        'include_dirs': ['DeepCL/EasyCL'],
#        'extra_compile_args': compile_options,
##        define_macros = [('EasyCL_EXPORTS',1)],
##        libraries = []
##        language='c++'
#        }
#    )
#]

ext_modules = [
#    Extension("_EasyCL",
#        sources = easyclsources + ['dummy_easycl.cpp'],
#        include_dirs = ['DeepCL/EasyCL'],
#        extra_compile_args=compile_options,
#        define_macros = [('EasyCL_EXPORTS',1),('MS_WIN32',1)],
##        libraries = []
##        language='c++'
#    )
#    Extension("libDeepCL",
#        list(map( lambda name : 'DeepCL/src/' + name, deepcl_sources)), # +
##            glob.glob('DeepCL/src/*.h'),
#        include_dirs = ['DeepCL/src','DeepCL/EasyCL'],
#        extra_compile_args = compile_options,
#        library_dirs = [ lib_build_dir() ],
#        libraries = [ "EasyCL" + get_so_suffix() ],
#        define_macros = [('DeepCL_EXPORTS',1)],
#        runtime_library_dirs=runtime_library_dirs
##        language='c++'
#    ),
    Extension("PyDeepCL",
              sources=["PyDeepCL.pyx", 'CyWrappers.cpp']
                + easyclsources
                + deepcl_sources,
#                glob.glob('DeepCL/EasyCL/*.h'),
              include_dirs = ['mysrc', 'mysrc/lua'],
              libraries= libraries,
              extra_compile_args=compile_options,
        define_macros = [('DeepCL_EXPORTS',1),('EasyCL_EXPORTS',1)],
#              extra_objects=['cDeepCL.pxd'],
#              library_dirs = [lib_build_dir()],
              runtime_library_dirs=runtime_library_dirs,
              language="c++"
    )
]

def read_if_exists(filename):
    filepath = os.path.join(os.path.dirname(__file__), filename)
    if os.path.isfile(filepath):
        return open(filepath).read()
    else:
        ""

version = read_if_exists('version.txt').strip().replace('v', '')
print('version: ', version )

setup(
  name = 'DeepCL',
  version = version,
#  version = "3.4.0rc1",  # synchronize to deepcl main version
  author = "Hugh Perkins",
  author_email = "hughperkins@gmail.com",
  description = 'python wrapper for DeepCL deep convolutional neural network library for OpenCL',
  license = 'MPL',
  url = 'https://github.com/hughperkins/DeepCL',
  long_description = read('README.rst'),
  classifiers = [
    'Development Status :: 4 - Beta',
    'Topic :: Scientific/Engineering :: Artificial Intelligence',
    'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
  ],
  install_requires = [],
  # install_requires = [],
  tests_require = ['nose>=1.3.4','Cython>=0.22','cogapp>=2.4','future>=0.14.3'],
  scripts = ['test_deepcl.py','test_lowlevel.py'],
 # modules = libraries,
#  lib raries = libraries,
  ext_modules = my_cythonize( ext_modules),
)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Safe Update: A simple service that waits for network access and tries to
# update every 10 minutes. It's intended to make the OP update process more
# robust against Git repository corruption. This service DOES NOT try to fix
# an already-corrupt BASEDIR Git repo, only prevent it from happening.
#
# During normal operation, both onroad and offroad, the update process makes
# no changes to the BASEDIR install of OP. All update attempts are performed
# in a disposable staging area provided by OverlayFS. It assumes the deleter
# process provides enough disk space to carry out the process.
#
# If an update succeeds, a flag is set, and the update is swapped in at the
# next reboot. If an update is interrupted or otherwise fails, the OverlayFS
# upper layer and metadata can be discarded before trying again.
#
# The swap on boot is triggered by launch_chffrplus.sh
# gated on the existence of $FINALIZED/.overlay_consistent and also the
# existence and mtime of $BASEDIR/.overlay_init.
#
# Other than build byproducts, BASEDIR should not be modified while this
# service is running. Developers modifying code directly in BASEDIR should
# disable this service.

import os
import datetime
import subprocess
import psutil
import shutil
import signal
import fcntl
import time
import threading
from pathlib import Path
from typing import List, Tuple, Optional

from common.basedir import BASEDIR
from common.params import Params
from selfdrive.hardware import EON, TICI
from selfdrive.swaglog import cloudlog
from selfdrive.controls.lib.alertmanager import set_offroad_alert

LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")

NEOS_VERSION = os.getenv("UPDATER_NEOS_VERSION", "/VERSION")
NEOSUPDATE_DIR = os.getenv("UPDATER_NEOSUPDATE_DIR", "/data/neoupdate")

OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata")
OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged")
FINALIZED = os.path.join(STAGING_ROOT, "finalized")

class WaitTimeHelper:
  def __init__(self, proc):
    self.proc = proc
    self.ready_event = threading.Event()
    self.shutdown = False
    signal.signal(signal.SIGTERM, self.graceful_shutdown)
    signal.signal(signal.SIGINT, self.graceful_shutdown)
    signal.signal(signal.SIGHUP, self.update_now)

  def graceful_shutdown(self, signum: int, frame) -> None:
    # umount -f doesn't appear effective in avoiding "device busy" on NEOS,
    # so don't actually die until the next convenient opportunity in main().
    cloudlog.info("caught SIGINT/SIGTERM, dismounting overlay at next opportunity")

    # forward the signal to all our child processes
    child_procs = self.proc.children(recursive=True)
    for p in child_procs:
      p.send_signal(signum)

    self.shutdown = True
    self.ready_event.set()

  def update_now(self, signum: int, frame) -> None:
    cloudlog.info("caught SIGHUP, running update check immediately")
    self.ready_event.set()

  def sleep(self, t: float) -> None:
    self.ready_event.wait(timeout=t)

def run(cmd: List[str], cwd: Optional[str] = None, low_priority: bool = False):
  if low_priority:
    cmd = ["nice", "-n", "19"] + cmd
  return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')

def set_consistent_flag(consistent: bool) -> None:
  os.sync()
  consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
  if consistent:
    consistent_file.touch()
  elif not consistent and consistent_file.exists():
    consistent_file.unlink()
  os.sync()

def set_params(new_version: bool, failed_count: int, exception: Optional[str]) -> None:
  params = Params()

  params.put("UpdateFailedCount", str(failed_count))
  if failed_count == 0:
    t = datetime.datetime.utcnow().isoformat()
    params.put("LastUpdateTime", t.encode('utf8'))

  if exception is None:
    params.delete("LastUpdateException")
  else:
    params.put("LastUpdateException", exception)

  if new_version:
    try:
      with open(os.path.join(FINALIZED, "RELEASES.md"), "rb") as f:
        r = f.read()
      r = r[:r.find(b'\n\n')]  # Slice latest release notes
      params.put("ReleaseNotes", r + b"\n")
    except Exception:
      params.put("ReleaseNotes", "")
    params.put("UpdateAvailable", "1")

def setup_git_options(cwd: str) -> None:
  # We sync FS object atimes (which NEOS doesn't use) and mtimes, but ctimes
  # are outside user control. Make sure Git is set up to ignore system ctimes,
  # because they change when we make hard links during finalize. Otherwise,
  # there is a lot of unnecessary churn. This appears to be a common need on
  # OSX as well: https://www.git-tower.com/blog/make-git-rebase-safe-on-osx/

  # We are using copytree to copy the directory, which also changes
  # inode numbers. Ignore those changes too.
  git_cfg = [
    ("core.trustctime", "false"),
    ("core.checkStat", "minimal"),
  ]
  for option, value in git_cfg:
    run(["git", "config", option, value], cwd)

def dismount_overlay() -> None:
  if os.path.ismount(OVERLAY_MERGED):
    cloudlog.info("unmounting existing overlay")
    args = ["umount", "-l", OVERLAY_MERGED]
    if TICI:
      args = ["sudo"] + args
    run(args)

def init_overlay() -> None:

  overlay_init_file = Path(os.path.join(BASEDIR, ".overlay_init"))

  # Re-create the overlay if BASEDIR/.git has changed since we created the overlay
  if overlay_init_file.is_file():
    git_dir_path = os.path.join(BASEDIR, ".git")
    new_files = run(["find", git_dir_path, "-newer", str(overlay_init_file)])
    if not len(new_files.splitlines()):
      # A valid overlay already exists
      return
    else:
      cloudlog.info(".git directory changed, recreating overlay")

  cloudlog.info("preparing new safe staging area")

  Params().put("UpdateAvailable", "0")
  set_consistent_flag(False)
  dismount_overlay()
  if os.path.isdir(STAGING_ROOT):
    shutil.rmtree(STAGING_ROOT)

  for dirname in [STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED]:
    os.mkdir(dirname, 0o755)

  if os.lstat(BASEDIR).st_dev != os.lstat(OVERLAY_MERGED).st_dev:
    raise RuntimeError("base and overlay merge directories are on different filesystems; not valid for overlay FS!")

  # Leave a timestamped canary in BASEDIR to check at startup. The device clock
  # should be correct by the time we get here. If the init file disappears, or
  # critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can
  # assume that BASEDIR has used for local development or otherwise modified,
  # and skips the update activation attempt.
  consistent_file = Path(os.path.join(BASEDIR, ".overlay_consistent"))
  if consistent_file.is_file():
    consistent_file.unlink()
  overlay_init_file.touch()

  os.sync()
  overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}"

  mount_cmd = ["mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED]
  if TICI:
    run(["sudo"] + mount_cmd)
    run(["sudo", "chmod", "755", os.path.join(OVERLAY_METADATA, "work")])
  else:
    run(mount_cmd)

def finalize_update() -> None:
  """Take the current OverlayFS merged view and finalize a copy outside of
  OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""

  # Remove the update ready flag and any old updates
  cloudlog.info("creating finalized version of the overlay")
  set_consistent_flag(False)

  # Copy the merged overlay view and set the update ready flag
  if os.path.exists(FINALIZED):
    shutil.rmtree(FINALIZED)
  shutil.copytree(OVERLAY_MERGED, FINALIZED, symlinks=True)

  set_consistent_flag(True)
  cloudlog.info("done finalizing overlay")

def handle_neos_update(wait_helper: WaitTimeHelper) -> None:
  with open(NEOS_VERSION, "r") as f:
    cur_neos = f.read().strip()

  updated_neos = run(["bash", "-c", r"unset REQUIRED_NEOS_VERSION && source launch_env.sh && \
                       echo -n $REQUIRED_NEOS_VERSION"], OVERLAY_MERGED).strip()

  cloudlog.info(f"NEOS version check: {cur_neos} vs {updated_neos}")
  if cur_neos == updated_neos:
    return

  cloudlog.info(f"Beginning background download for NEOS {updated_neos}")
  set_offroad_alert("Offroad_NeosUpdate", True)

  updater_path = os.path.join(OVERLAY_MERGED, "installer/updater/updater")
  update_manifest = f"file://{OVERLAY_MERGED}/installer/updater/update.json"

  neos_downloaded = False
  start_time = time.monotonic()
  # Try to download for one day
  while not neos_downloaded and not wait_helper.shutdown and \
        (time.monotonic() - start_time < 60*60*24):
    wait_helper.ready_event.clear()
    try:
      run([updater_path, "bgcache", update_manifest], OVERLAY_MERGED, low_priority=True)
      neos_downloaded = True
    except subprocess.CalledProcessError:
      cloudlog.info("NEOS background download failed, retrying")
      wait_helper.sleep(120)

  # If the download failed, we'll show the alert again when we retry
  set_offroad_alert("Offroad_NeosUpdate", False)
  if not neos_downloaded:
    raise Exception("Failed to download NEOS update")
  cloudlog.info(f"NEOS background download successful, took {time.monotonic() - start_time} seconds")

def check_git_fetch_result(fetch_txt):
  err_msg = "Failed to add the host to the list of known hosts (/data/data/com.termux/files/home/.ssh/known_hosts).\n"
  return len(fetch_txt) > 0 and (fetch_txt != err_msg)

def check_for_update() -> Tuple[bool, bool]:
  setup_git_options(OVERLAY_MERGED)
  try:
    git_fetch_output = run(["git", "fetch", "--dry-run"], OVERLAY_MERGED, low_priority=True)
    return True, check_git_fetch_result(git_fetch_output)
  except subprocess.CalledProcessError:
    return False, False

def fetch_update(wait_helper: WaitTimeHelper) -> bool:
  cloudlog.info("attempting git fetch inside staging overlay")

  setup_git_options(OVERLAY_MERGED)

  git_fetch_output = run(["git", "fetch"], OVERLAY_MERGED, low_priority=True)
  cloudlog.info("git fetch success: %s", git_fetch_output)

  cur_hash = run(["git", "rev-parse", "HEAD"], OVERLAY_MERGED).rstrip()
  upstream_hash = run(["git", "rev-parse", "@{u}"], OVERLAY_MERGED).rstrip()
  new_version = cur_hash != upstream_hash
  git_fetch_result = check_git_fetch_result(git_fetch_output)

  cloudlog.info("comparing %s to %s" % (cur_hash, upstream_hash))
  if new_version or git_fetch_result:
    cloudlog.info("Running update")

    if new_version:
      cloudlog.info("git reset in progress")
      r = [
        run(["git", "reset", "--hard", "@{u}"], OVERLAY_MERGED, low_priority=True),
        run(["git", "clean", "-xdf"], OVERLAY_MERGED, low_priority=True ),
        run(["git", "submodule", "init"], OVERLAY_MERGED, low_priority=True),
        run(["git", "submodule", "update"], OVERLAY_MERGED, low_priority=True),
      ]
      cloudlog.info("git reset success: %s", '\n'.join(r))

      if EON:
        handle_neos_update(wait_helper)

    # Create the finalized, ready-to-swap update
    finalize_update()
    cloudlog.info("openpilot update successful!")
  else:
    cloudlog.info("nothing new from git at this time")

  return new_version

def main():
  params = Params()

  if params.get("DisableUpdates") == b"1":
    raise RuntimeError("updates are disabled by the DisableUpdates param")

  # TODO: enable this before 0.8.1 release
  #leon = "letv" in open("/proc/cmdline").read()
  #if EON and not leon:
  #  raise RuntimeError("updates are disabled due to device deprecation")

  if EON and os.geteuid() != 0:
    raise RuntimeError("updated must be launched as root!")

  # Set low io priority
  proc = psutil.Process()
  if psutil.LINUX:
    proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)

  ov_lock_fd = open(LOCK_FILE, 'w')
  try:
    fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  except IOError as e:
    raise RuntimeError("couldn't get overlay lock; is another updated running?") from e

  # Wait for IsOffroad to be set before our first update attempt
  wait_helper = WaitTimeHelper(proc)
  wait_helper.sleep(30)

  overlay_init = Path(os.path.join(BASEDIR, ".overlay_init"))
  if overlay_init.exists():
    overlay_init.unlink()

  first_run = True
  last_fetch_time = 0
  update_failed_count = 0

  # Run the update loop
  #  * every 1m, do a lightweight internet/update check
  #  * every 10m, do a full git fetch
  while not wait_helper.shutdown:
    update_now = wait_helper.ready_event.is_set()
    wait_helper.ready_event.clear()

    # Don't run updater while onroad or if the time's wrong
    time_wrong = datetime.datetime.utcnow().year < 2019
    is_onroad = params.get("IsOffroad") != b"1"
    if is_onroad or time_wrong:
      wait_helper.sleep(30)
      cloudlog.info("not running updater, not offroad")
      continue

    # Attempt an update
    exception = None
    new_version = False
    update_failed_count += 1
    try:
      init_overlay()

      internet_ok, update_available = check_for_update()
      if internet_ok and not update_available:
        update_failed_count = 0

      # Fetch updates at most every 10 minutes
      if internet_ok and (update_now or time.monotonic() - last_fetch_time > 60*10):
        new_version = fetch_update(wait_helper)
        update_failed_count = 0
        last_fetch_time = time.monotonic()

        if first_run and not new_version and os.path.isdir(NEOSUPDATE_DIR):
          shutil.rmtree(NEOSUPDATE_DIR)
        first_run = False
    except subprocess.CalledProcessError as e:
      cloudlog.event(
        "update process failed",
        cmd=e.cmd,
        output=e.output,
        returncode=e.returncode
      )
      exception = f"command failed: {e.cmd}\n{e.output}"
    except Exception as e:
      cloudlog.exception("uncaught updated exception, shouldn't happen")
      exception = str(e)

    set_params(new_version, update_failed_count, exception)
    wait_helper.sleep(60)

  dismount_overlay()

if __name__ == "__main__":
  main()
<commit_msg>disable updater for oneplus EONs<commit_after>#!/usr/bin/env python3

# Safe Update: A simple service that waits for network access and tries to
# update every 10 minutes. It's intended to make the OP update process more
# robust against Git repository corruption. This service DOES NOT try to fix
# an already-corrupt BASEDIR Git repo, only prevent it from happening.
#
# During normal operation, both onroad and offroad, the update process makes
# no changes to the BASEDIR install of OP. All update attempts are performed
# in a disposable staging area provided by OverlayFS. It assumes the deleter
# process provides enough disk space to carry out the process.
#
# If an update succeeds, a flag is set, and the update is swapped in at the
# next reboot. If an update is interrupted or otherwise fails, the OverlayFS
# upper layer and metadata can be discarded before trying again.
#
# The swap on boot is triggered by launch_chffrplus.sh
# gated on the existence of $FINALIZED/.overlay_consistent and also the
# existence and mtime of $BASEDIR/.overlay_init.
#
# Other than build byproducts, BASEDIR should not be modified while this
# service is running. Developers modifying code directly in BASEDIR should
# disable this service.

import os
import datetime
import subprocess
import psutil
import shutil
import signal
import fcntl
import time
import threading
from pathlib import Path
from typing import List, Tuple, Optional

from common.basedir import BASEDIR
from common.params import Params
from selfdrive.hardware import EON, TICI
from selfdrive.swaglog import cloudlog
from selfdrive.controls.lib.alertmanager import set_offroad_alert

LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")

NEOS_VERSION = os.getenv("UPDATER_NEOS_VERSION", "/VERSION")
NEOSUPDATE_DIR = os.getenv("UPDATER_NEOSUPDATE_DIR", "/data/neoupdate")

OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata")
OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged")
FINALIZED = os.path.join(STAGING_ROOT, "finalized")

class WaitTimeHelper:
  def __init__(self, proc):
    self.proc = proc
    self.ready_event = threading.Event()
    self.shutdown = False
    signal.signal(signal.SIGTERM, self.graceful_shutdown)
    signal.signal(signal.SIGINT, self.graceful_shutdown)
    signal.signal(signal.SIGHUP, self.update_now)

  def graceful_shutdown(self, signum: int, frame) -> None:
    # umount -f doesn't appear effective in avoiding "device busy" on NEOS,
    # so don't actually die until the next convenient opportunity in main().
    cloudlog.info("caught SIGINT/SIGTERM, dismounting overlay at next opportunity")

    # forward the signal to all our child processes
    child_procs = self.proc.children(recursive=True)
    for p in child_procs:
      p.send_signal(signum)

    self.shutdown = True
    self.ready_event.set()

  def update_now(self, signum: int, frame) -> None:
    cloudlog.info("caught SIGHUP, running update check immediately")
    self.ready_event.set()

  def sleep(self, t: float) -> None:
    self.ready_event.wait(timeout=t)

def run(cmd: List[str], cwd: Optional[str] = None, low_priority: bool = False):
  if low_priority:
    cmd = ["nice", "-n", "19"] + cmd
  return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')

def set_consistent_flag(consistent: bool) -> None:
  os.sync()
  consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
  if consistent:
    consistent_file.touch()
  elif not consistent and consistent_file.exists():
    consistent_file.unlink()
  os.sync()

def set_params(new_version: bool, failed_count: int, exception: Optional[str]) -> None:
  params = Params()

  params.put("UpdateFailedCount", str(failed_count))
  if failed_count == 0:
    t = datetime.datetime.utcnow().isoformat()
    params.put("LastUpdateTime", t.encode('utf8'))

  if exception is None:
    params.delete("LastUpdateException")
  else:
    params.put("LastUpdateException", exception)

  if new_version:
    try:
      with open(os.path.join(FINALIZED, "RELEASES.md"), "rb") as f:
        r = f.read()
      r = r[:r.find(b'\n\n')]  # Slice latest release notes
      params.put("ReleaseNotes", r + b"\n")
    except Exception:
      params.put("ReleaseNotes", "")
    params.put("UpdateAvailable", "1")

def setup_git_options(cwd: str) -> None:
  # We sync FS object atimes (which NEOS doesn't use) and mtimes, but ctimes
  # are outside user control. Make sure Git is set up to ignore system ctimes,
  # because they change when we make hard links during finalize. Otherwise,
  # there is a lot of unnecessary churn. This appears to be a common need on
  # OSX as well: https://www.git-tower.com/blog/make-git-rebase-safe-on-osx/

  # We are using copytree to copy the directory, which also changes
  # inode numbers. Ignore those changes too.
  git_cfg = [
    ("core.trustctime", "false"),
    ("core.checkStat", "minimal"),
  ]
  for option, value in git_cfg:
    run(["git", "config", option, value], cwd)

def dismount_overlay() -> None:
  if os.path.ismount(OVERLAY_MERGED):
    cloudlog.info("unmounting existing overlay")
    args = ["umount", "-l", OVERLAY_MERGED]
    if TICI:
      args = ["sudo"] + args
    run(args)

def init_overlay() -> None:

  overlay_init_file = Path(os.path.join(BASEDIR, ".overlay_init"))

  # Re-create the overlay if BASEDIR/.git has changed since we created the overlay
  if overlay_init_file.is_file():
    git_dir_path = os.path.join(BASEDIR, ".git")
    new_files = run(["find", git_dir_path, "-newer", str(overlay_init_file)])
    if not len(new_files.splitlines()):
      # A valid overlay already exists
      return
    else:
      cloudlog.info(".git directory changed, recreating overlay")

  cloudlog.info("preparing new safe staging area")

  Params().put("UpdateAvailable", "0")
  set_consistent_flag(False)
  dismount_overlay()
  if os.path.isdir(STAGING_ROOT):
    shutil.rmtree(STAGING_ROOT)

  for dirname in [STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED]:
    os.mkdir(dirname, 0o755)

  if os.lstat(BASEDIR).st_dev != os.lstat(OVERLAY_MERGED).st_dev:
    raise RuntimeError("base and overlay merge directories are on different filesystems; not valid for overlay FS!")

  # Leave a timestamped canary in BASEDIR to check at startup. The device clock
  # should be correct by the time we get here. If the init file disappears, or
  # critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can
  # assume that BASEDIR has used for local development or otherwise modified,
  # and skips the update activation attempt.
  consistent_file = Path(os.path.join(BASEDIR, ".overlay_consistent"))
  if consistent_file.is_file():
    consistent_file.unlink()
  overlay_init_file.touch()

  os.sync()
  overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}"

  mount_cmd = ["mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED]
  if TICI:
    run(["sudo"] + mount_cmd)
    run(["sudo", "chmod", "755", os.path.join(OVERLAY_METADATA, "work")])
  else:
    run(mount_cmd)

def finalize_update() -> None:
  """Take the current OverlayFS merged view and finalize a copy outside of
  OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""

  # Remove the update ready flag and any old updates
  cloudlog.info("creating finalized version of the overlay")
  set_consistent_flag(False)

  # Copy the merged overlay view and set the update ready flag
  if os.path.exists(FINALIZED):
    shutil.rmtree(FINALIZED)
  shutil.copytree(OVERLAY_MERGED, FINALIZED, symlinks=True)

  set_consistent_flag(True)
  cloudlog.info("done finalizing overlay")

def handle_neos_update(wait_helper: WaitTimeHelper) -> None:
  with open(NEOS_VERSION, "r") as f:
    cur_neos = f.read().strip()

  updated_neos = run(["bash", "-c", r"unset REQUIRED_NEOS_VERSION && source launch_env.sh && \
                       echo -n $REQUIRED_NEOS_VERSION"], OVERLAY_MERGED).strip()

  cloudlog.info(f"NEOS version check: {cur_neos} vs {updated_neos}")
  if cur_neos == updated_neos:
    return

  cloudlog.info(f"Beginning background download for NEOS {updated_neos}")
  set_offroad_alert("Offroad_NeosUpdate", True)

  updater_path = os.path.join(OVERLAY_MERGED, "installer/updater/updater")
  update_manifest = f"file://{OVERLAY_MERGED}/installer/updater/update.json"

  neos_downloaded = False
  start_time = time.monotonic()
  # Try to download for one day
  while not neos_downloaded and not wait_helper.shutdown and \
        (time.monotonic() - start_time < 60*60*24):
    wait_helper.ready_event.clear()
    try:
      run([updater_path, "bgcache", update_manifest], OVERLAY_MERGED, low_priority=True)
      neos_downloaded = True
    except subprocess.CalledProcessError:
      cloudlog.info("NEOS background download failed, retrying")
      wait_helper.sleep(120)

  # If the download failed, we'll show the alert again when we retry
  set_offroad_alert("Offroad_NeosUpdate", False)
  if not neos_downloaded:
    raise Exception("Failed to download NEOS update")
  cloudlog.info(f"NEOS background download successful, took {time.monotonic() - start_time} seconds")

def check_git_fetch_result(fetch_txt):
  err_msg = "Failed to add the host to the list of known hosts (/data/data/com.termux/files/home/.ssh/known_hosts).\n"
  return len(fetch_txt) > 0 and (fetch_txt != err_msg)

def check_for_update() -> Tuple[bool, bool]:
  setup_git_options(OVERLAY_MERGED)
  try:
    git_fetch_output = run(["git", "fetch", "--dry-run"], OVERLAY_MERGED, low_priority=True)
    return True, check_git_fetch_result(git_fetch_output)
  except subprocess.CalledProcessError:
    return False, False

def fetch_update(wait_helper: WaitTimeHelper) -> bool:
  cloudlog.info("attempting git fetch inside staging overlay")

  setup_git_options(OVERLAY_MERGED)

  git_fetch_output = run(["git", "fetch"], OVERLAY_MERGED, low_priority=True)
  cloudlog.info("git fetch success: %s", git_fetch_output)

  cur_hash = run(["git", "rev-parse", "HEAD"], OVERLAY_MERGED).rstrip()
  upstream_hash = run(["git", "rev-parse", "@{u}"], OVERLAY_MERGED).rstrip()
  new_version = cur_hash != upstream_hash
  git_fetch_result = check_git_fetch_result(git_fetch_output)

  cloudlog.info("comparing %s to %s" % (cur_hash, upstream_hash))
  if new_version or git_fetch_result:
    cloudlog.info("Running update")

    if new_version:
      cloudlog.info("git reset in progress")
      r = [
        run(["git", "reset", "--hard", "@{u}"], OVERLAY_MERGED, low_priority=True),
        run(["git", "clean", "-xdf"], OVERLAY_MERGED, low_priority=True ),
        run(["git", "submodule", "init"], OVERLAY_MERGED, low_priority=True),
        run(["git", "submodule", "update"], OVERLAY_MERGED, low_priority=True),
      ]
      cloudlog.info("git reset success: %s", '\n'.join(r))

      if EON:
        handle_neos_update(wait_helper)

    # Create the finalized, ready-to-swap update
    finalize_update()
    cloudlog.info("openpilot update successful!")
  else:
    cloudlog.info("nothing new from git at this time")

  return new_version

def main():
  params = Params()

  if params.get("DisableUpdates") == b"1":
    raise RuntimeError("updates are disabled by the DisableUpdates param")

  # TODO: remove this after next release
  if EON and "letv" not in open("/proc/cmdline").read():
    raise RuntimeError("updates are disabled due to device deprecation")

  if EON and os.geteuid() != 0:
    raise RuntimeError("updated must be launched as root!")

  # Set low io priority
  proc = psutil.Process()
  if psutil.LINUX:
    proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)

  ov_lock_fd = open(LOCK_FILE, 'w')
  try:
    fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  except IOError as e:
    raise RuntimeError("couldn't get overlay lock; is another updated running?") from e

  # Wait for IsOffroad to be set before our first update attempt
  wait_helper = WaitTimeHelper(proc)
  wait_helper.sleep(30)

  overlay_init = Path(os.path.join(BASEDIR, ".overlay_init"))
  if overlay_init.exists():
    overlay_init.unlink()

  first_run = True
  last_fetch_time = 0
  update_failed_count = 0

  # Run the update loop
  #  * every 1m, do a lightweight internet/update check
  #  * every 10m, do a full git fetch
  while not wait_helper.shutdown:
    update_now = wait_helper.ready_event.is_set()
    wait_helper.ready_event.clear()

    # Don't run updater while onroad or if the time's wrong
    time_wrong = datetime.datetime.utcnow().year < 2019
    is_onroad = params.get("IsOffroad") != b"1"
    if is_onroad or time_wrong:
      wait_helper.sleep(30)
      cloudlog.info("not running updater, not offroad")
      continue

    # Attempt an update
    exception = None
    new_version = False
    update_failed_count += 1
    try:
      init_overlay()

      internet_ok, update_available = check_for_update()
      if internet_ok and not update_available:
        update_failed_count = 0

      # Fetch updates at most every 10 minutes
      if internet_ok and (update_now or time.monotonic() - last_fetch_time > 60*10):
        new_version = fetch_update(wait_helper)
        update_failed_count = 0
        last_fetch_time = time.monotonic()

        if first_run and not new_version and os.path.isdir(NEOSUPDATE_DIR):
          shutil.rmtree(NEOSUPDATE_DIR)
        first_run = False
    except subprocess.CalledProcessError as e:
      cloudlog.event(
        "update process failed",
        cmd=e.cmd,
        output=e.output,
        returncode=e.returncode
      )
      exception = f"command failed: {e.cmd}\n{e.output}"
    except Exception as e:
      cloudlog.exception("uncaught updated exception, shouldn't happen")
      exception = str(e)

    set_params(new_version, update_failed_count, exception)
    wait_helper.sleep(60)

  dismount_overlay()

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>VERSION = (0, 3, 8)
__version__ = '.'.join(map(str, VERSION))

import os.path
from mimetypes import guess_type

def _lazy_load(fn):
    _cached = []
    def _decorated():
        if not _cached:
            _cached.append(fn())
        return _cached[0]
    def clear():
        while _cached:
            _cached.pop()
    _decorated.clear = clear
    return _decorated

@_lazy_load
def _get_sendfile():
    try:
        from importlib import import_module
    except ImportError:
        from django.utils.importlib import import_module
    from django.conf import settings
    from django.core.exceptions import ImproperlyConfigured

    backend = getattr(settings, 'SENDFILE_BACKEND', None)
    if not backend:
        raise ImproperlyConfigured('You must specify a value for SENDFILE_BACKEND')
    module = import_module(backend)
    return module.sendfile

def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None):
    '''
    create a response to send file using backend configured in SENDFILE_BACKEND

    If attachment is True the content-disposition header will be set.
    This will typically prompt the user to download the file, rather
    than view it.  The content-disposition filename depends on the
    value of attachment_filename:

        None (default): Same as filename
        False: No content-disposition filename
        String: Value used as filename

    If no mimetype or encoding are specified, then they will be guessed via the
    filename (using the standard python mimetypes module)
    '''
    _sendfile = _get_sendfile()

    if not os.path.exists(filename):
        from django.http import Http404
        raise Http404('"%s" does not exist' % filename)

    guessed_mimetype, guessed_encoding = guess_type(filename)
    if mimetype is None:
        if guessed_mimetype:
            mimetype = guessed_mimetype
        else:
            mimetype = 'application/octet-stream'

    response = _sendfile(request, filename, mimetype=mimetype)
    if attachment:
        if attachment_filename is None:
            attachment_filename = os.path.basename(filename)
        parts = ['attachment']
        if attachment_filename:
            from unidecode import unidecode
            try:
                from django.utils.encoding import force_text
            except ImportError:
                # Django 1.3
                from django.utils.encoding import force_unicode as force_text
            attachment_filename = force_text(attachment_filename)
            ascii_filename = unidecode(attachment_filename)
            parts.append('filename="%s"' % ascii_filename)
            if ascii_filename != attachment_filename:
                from django.utils.http import urlquote
                quoted_filename = urlquote(attachment_filename)
                parts.append('filename*=UTF-8\'\'%s' % quoted_filename)
        response['Content-Disposition'] = '; '.join(parts)

    response['Content-length'] = os.path.getsize(filename)
    response['Content-Type'] = mimetype
    if not encoding:
        encoding = guessed_encoding
    if encoding:
        response['Content-Encoding'] = encoding

    return response
<commit_msg>Version 0.3.9<commit_after>VERSION = (0, 3, 9)
__version__ = '.'.join(map(str, VERSION))

import os.path
from mimetypes import guess_type

def _lazy_load(fn):
    _cached = []
    def _decorated():
        if not _cached:
            _cached.append(fn())
        return _cached[0]
    def clear():
        while _cached:
            _cached.pop()
    _decorated.clear = clear
    return _decorated

@_lazy_load
def _get_sendfile():
    try:
        from importlib import import_module
    except ImportError:
        from django.utils.importlib import import_module
    from django.conf import settings
    from django.core.exceptions import ImproperlyConfigured

    backend = getattr(settings, 'SENDFILE_BACKEND', None)
    if not backend:
        raise ImproperlyConfigured('You must specify a value for SENDFILE_BACKEND')
    module = import_module(backend)
    return module.sendfile

def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None):
    '''
    create a response to send file using backend configured in SENDFILE_BACKEND

    If attachment is True the content-disposition header will be set.
    This will typically prompt the user to download the file, rather
    than view it.  The content-disposition filename depends on the
    value of attachment_filename:

        None (default): Same as filename
        False: No content-disposition filename
        String: Value used as filename

    If no mimetype or encoding are specified, then they will be guessed via the
    filename (using the standard python mimetypes module)
    '''
    _sendfile = _get_sendfile()

    if not os.path.exists(filename):
        from django.http import Http404
        raise Http404('"%s" does not exist' % filename)

    guessed_mimetype, guessed_encoding = guess_type(filename)
    if mimetype is None:
        if guessed_mimetype:
            mimetype = guessed_mimetype
        else:
            mimetype = 'application/octet-stream'

    response = _sendfile(request, filename, mimetype=mimetype)
    if attachment:
        if attachment_filename is None:
            attachment_filename = os.path.basename(filename)
        parts = ['attachment']
        if attachment_filename:
            from unidecode import unidecode
            try:
                from django.utils.encoding import force_text
            except ImportError:
                # Django 1.3
                from django.utils.encoding import force_unicode as force_text
            attachment_filename = force_text(attachment_filename)
            ascii_filename = unidecode(attachment_filename)
            parts.append('filename="%s"' % ascii_filename)
            if ascii_filename != attachment_filename:
                from django.utils.http import urlquote
                quoted_filename = urlquote(attachment_filename)
                parts.append('filename*=UTF-8\'\'%s' % quoted_filename)
        response['Content-Disposition'] = '; '.join(parts)

    response['Content-length'] = os.path.getsize(filename)
    response['Content-Type'] = mimetype
    if not encoding:
        encoding = guessed_encoding
    if encoding:
        response['Content-Encoding'] = encoding

    return response
<|endoftext|>


<commit_before>import os
import time
import redis
import threading
import ConfigParser
import datetime
import paho.mqtt.client as     mqtt
from pytz               import timezone
from Chromecast         import Chromecast
from InformationFetcher import InformationFetcher

class AlarmClock(threading.Thread):

    def _readConfig(self):
        update = False

        if not os.path.isdir(self._homeDir):
            print "Creating homeDir"
            os.makedirs(self._homeDir)

        if os.path.isfile(self._configFileName):
            self._config.read(self._configFileName)
        else:
            print "Config file not found"
            update = True

        if not self._config.has_section('REDIS'):
            print "Adding Redis part"
            update = True
            self._config.add_section("REDIS")

        if not self._config.has_option("REDIS", "ServerAddress"):
            print "No Server Address"
            update = True
            self._config.set("REDIS", "ServerAddress", "<ServerAddress>")

        if not self._config.has_option("REDIS", "ServerPort"):
            print "No Server Port"
            update = True
            self._config.set("REDIS", "ServerPort", "6379")

        if not self._config.has_section('MQTT'):
            print "Adding MQTT part"
            update = True
            self._config.add_section("MQTT")

        if not self._config.has_option("MQTT", "ServerAddress"):
            print "No Server Address"
            update = True
            self._config.set("MQTT", "ServerAddress", "<ServerAddress>")

        if not self._config.has_option("MQTT", "ServerPort"):
            print "No Server Port"
            update = True
            self._config.set("MQTT", "ServerPort", "1883")

        if update:
            with open(self._configFileName, 'w') as f:
                self._config.write(f)

    def __init__(self):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self._info                   = InformationFetcher()
        self._homeDir                = os.path.expanduser("~/.sensomatic")
        self._configFileName         = self._homeDir + '/config.ini'
        self._config                 = ConfigParser.ConfigParser()
        self._readConfig()
        self._redis                  = redis.StrictRedis(host=self._config.get("REDIS", "ServerAddress"), port=self._config.get("REDIS", "ServerPort"), db=0)
        self._mqclient               = mqtt.Client("AlarmClock", clean_session=True)
        self._mqclient.on_connect    = self._on_connect
        self._mqclient.on_message    = self._on_message
        self._mqclient.on_disconnect = self._on_disconnect
        self._mqclient.connect(self._config.get("MQTT", "ServerAddress"), self._config.get("MQTT", "ServerPort"), 60)
        time.sleep(1)

    def _on_connect(self, client, userdata, rc, msg):
        print "Connected with result code %s" % rc

    def _on_message(self, client, userdata, msg):
        print "Mq Received on channel %s -> %s" % (msg.topic, msg.payload)

    def _on_disconnect(self, client, userdata, msg):
        print "Disconnect Alarmclock"

    def run(self):
        starttime, endtime = self._info.getNextWackeuptime()
        updated            = time.time()
        waking             = False
        music              = False

        while True:
            self._mqclient.loop(max_packets=100)
            diff = (starttime - datetime.datetime.now(timezone('Europe/Berlin'))).total_seconds()

            #Switch on the light 15 min before event
            if 0 < diff < (60 * 15):
                print "light"
                lightlevel = int((1.0 - (diff / (60 * 15))) * 100)
                self._mqclient.publish("ansiroom/bedlight/sleep/sunrise", lightlevel)
                waking = True
                self._mqclient.loop(max_packets=100)

            #5 Min before slowly turn on the music
            if 0 < diff < (60 * 5):
                if not music:
                    Chromecast().volume('Chromeansi', 0)
                    Chromecast().playMusicURL('Chromeansi', "http://inforadio.de/livemp3")
                    music = True
                volume = (1.0 - (diff / (60 * 5))) * 0.6
                Chromecast().volume('Chromeansi', volume)

            if diff < 0 and waking:
                self._mqclient.publish("ansiroom/bedlight/sleep/sunrise", 100)
                Chromecast().volume('Chromeansi', 0.6)
                waking = False
                music  = False
                print "ENDE"
                self._redis.setex("ansiwakeup", 60 * 60 * 5, time.time())

            if (time.time() - updated) > (60 * 15):
                starttime, endtime = self._info.getNextWackeuptime()
                updated = time.time()

            self._mqclient.loop(max_packets=100)
            time.sleep(1)

if __name__ == '__main__':
    a = AlarmClock()
    a.start()
    time.sleep(420)<commit_msg>longer interval<commit_after>import os
import time
import redis
import threading
import ConfigParser
import datetime
import paho.mqtt.client as     mqtt
from pytz               import timezone
from Chromecast         import Chromecast
from InformationFetcher import InformationFetcher

class AlarmClock(threading.Thread):

    def _readConfig(self):
        update = False

        if not os.path.isdir(self._homeDir):
            print "Creating homeDir"
            os.makedirs(self._homeDir)

        if os.path.isfile(self._configFileName):
            self._config.read(self._configFileName)
        else:
            print "Config file not found"
            update = True

        if not self._config.has_section('REDIS'):
            print "Adding Redis part"
            update = True
            self._config.add_section("REDIS")

        if not self._config.has_option("REDIS", "ServerAddress"):
            print "No Server Address"
            update = True
            self._config.set("REDIS", "ServerAddress", "<ServerAddress>")

        if not self._config.has_option("REDIS", "ServerPort"):
            print "No Server Port"
            update = True
            self._config.set("REDIS", "ServerPort", "6379")

        if not self._config.has_section('MQTT'):
            print "Adding MQTT part"
            update = True
            self._config.add_section("MQTT")

        if not self._config.has_option("MQTT", "ServerAddress"):
            print "No Server Address"
            update = True
            self._config.set("MQTT", "ServerAddress", "<ServerAddress>")

        if not self._config.has_option("MQTT", "ServerPort"):
            print "No Server Port"
            update = True
            self._config.set("MQTT", "ServerPort", "1883")

        if update:
            with open(self._configFileName, 'w') as f:
                self._config.write(f)

    def __init__(self):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self._info                   = InformationFetcher()
        self._homeDir                = os.path.expanduser("~/.sensomatic")
        self._configFileName         = self._homeDir + '/config.ini'
        self._config                 = ConfigParser.ConfigParser()
        self._readConfig()
        self._redis                  = redis.StrictRedis(host=self._config.get("REDIS", "ServerAddress"), port=self._config.get("REDIS", "ServerPort"), db=0)
        self._mqclient               = mqtt.Client("AlarmClock", clean_session=True)
        self._mqclient.on_connect    = self._on_connect
        self._mqclient.on_message    = self._on_message
        self._mqclient.on_disconnect = self._on_disconnect
        self._mqclient.connect(self._config.get("MQTT", "ServerAddress"), self._config.get("MQTT", "ServerPort"), 60)
        time.sleep(1)

    def _on_connect(self, client, userdata, rc, msg):
        print "Connected with result code %s" % rc

    def _on_message(self, client, userdata, msg):
        print "Mq Received on channel %s -> %s" % (msg.topic, msg.payload)

    def _on_disconnect(self, client, userdata, msg):
        print "Disconnect Alarmclock"

    def run(self):
        starttime, endtime = self._info.getNextWackeuptime()
        updated            = time.time()
        waking             = False
        music              = False

        while True:
            self._mqclient.loop(max_packets=100)
            diff = (starttime - datetime.datetime.now(timezone('Europe/Berlin'))).total_seconds()

            #Switch on the light 15 min before event
            if 0 < diff < (60 * 15):
                print "light"
                lightlevel = int((1.0 - (diff / (60 * 15))) * 100)
                self._mqclient.publish("ansiroom/bedlight/sleep/sunrise", lightlevel)
                waking = True
                self._mqclient.loop(max_packets=100)

            #5 Min before slowly turn on the music
            if 0 < diff < (60 * 5):
                if not music:
                    Chromecast().volume('Chromeansi', 0)
                    Chromecast().playMusicURL('Chromeansi', "http://inforadio.de/livemp3")
                    music = True
                volume = (1.0 - (diff / (60 * 5))) * 0.6
                Chromecast().volume('Chromeansi', volume)

            if diff < 0 and waking:
                self._mqclient.publish("ansiroom/bedlight/sleep/sunrise", 100)
                Chromecast().volume('Chromeansi', 0.6)
                waking = False
                music  = False
                print "ENDE"
                self._redis.setex("ansiwakeup", 60 * 60 * 5, time.time())

            if (time.time() - updated) > (60 * 15):
                starttime, endtime = self._info.getNextWackeuptime()
                updated = time.time()

            self._mqclient.loop(max_packets=100)
            time.sleep(5)

if __name__ == '__main__':
    a = AlarmClock()
    a.start()
    time.sleep(5)<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import re
import os
import sys
import json
import pytest
import logging
import traceback
import logging_tools
from collections import OrderedDict
import unittest
from StringIO import StringIO
import __builtin__

from plugin import PytestPlugin

logger = logging_tools.get_logger(__name__)

class RollbackImporter:
    def __init__(self):
        "Creates an instance and installs as the global importer"
        self.previousModules = sys.modules.copy()
        self.realImport = __builtin__.__import__
        __builtin__.__import__ = self._import
        self.newModules = {}

    def _import(self, name, globals=None, locals=None, fromlist=[], *args, **kwargs):
        # logger.debug('args: %s', args)
        # logger.debug('kwargs: %s', kwargs)
        result = apply(self.realImport, (name, globals, locals, fromlist))
        self.newModules[name] = 1
        return result

    def uninstall(self):
        logger.debug('uninstalling modules')
        for modname in self.newModules.keys():
            logger.debug('modname: %s', modname)
            if not self.previousModules.has_key(modname):
                # Force reload when modname next imported
                del(sys.modules[modname])
        __builtin__.__import__ = self.realImport

class Runner(object):
    def __init__(self, path='.', write_pipe=None, pipe_size=None, pipe_semaphore=None):
        self.ui = None
        self.path = path
        self.tests = OrderedDict()
        logger.debug('%s Init', self.__class__.__name__)
        self.write_pipe = os.fdopen(write_pipe, 'w', 0)
        self.pipe_size = pipe_size
        self.pipe_semaphore = pipe_semaphore

    def pipe_send(self, method, **kwargs):
        data = '%s\n' % json.dumps({
                'method': method,
                'params': kwargs
        })
        pipe_lock = self.pipe_size.get_lock()
        data_size = len(data)
        logger.debug('writing to pipe size: %s, pipe_size: %s',
                     data_size, self.pipe_size.value)

        if self.pipe_size.value + data_size >= 4096:
            logger.debug('waiting for reader')
            self.pipe_semaphore.clear()
            self.pipe_semaphore.wait()
            logger.debug('reader finished')

        with pipe_lock:
            self.pipe_size.value += data_size
            self.write_pipe.write(data)

    def set_test_result(self, test_id, report, output):
        self.pipe_send('set_test_result',
            test_id=test_id,
            output=output,
            result_state=self.result_state(report),
            when=report.when,
            outcome=report.outcome
        )

    def set_test_state(self, test_id, state):
        self.pipe_send('set_test_state',
            test_id=test_id,
            state=state
        )

    def set_exception_info(self, test_id, excinfo, when):
        exc_type, exc_value, exc_traceback = excinfo._excinfo
        extracted_traceback = traceback.extract_tb(exc_traceback)
        self.pipe_send('set_exception_info',
            test_id=test_id,
            exc_type='TODO',
            exc_value=traceback.format_exception_only(exc_type, exc_value)[-1],
            extracted_traceback=extracted_traceback,
            result='failed',
            when=when
        )

    def get_test_id(self, test):
        raise NotImplementedError()

class UnittestRunner(Runner):
    def get_suite_tests(suite):
        test_list = {}
        for item in suite:
            if isinstance(item, unittest.suite.TestSuite):
                test_list.update(self.get_suite_tests(item))
            else:
                test_list[item.id()] = item

        return OrderedDict(sorted(test_list.iteritems()))

    def get_test_id(self, test):
        return test.id()

    def init_tests(self):
        loader = unittest.TestLoader()
        top_suite = loader.discover(self.path)
        self.tests = self.get_suite_tests(top_suite)

    def reload_tests(self):
        if self.rollbackImporter:
            self.rollbackImporter.uninstall()
        self.rollbackImporter = RollbackImporter()
        self.init_tests()

    def run_tests(self, failed_only=True, filtered=True):
        self.reload_tests()
        tests = self._get_tests(failed_only, filtered)

        for test_id, suite in tests.iteritems():
            self._run_test(test_id)

    def result_state(self, test_result):
        if not test_result:
            return ''
        elif test_result.skipped:
            return 'skipped'
        elif test_result.failures:
            return 'failed'
        elif test_result.errors:
            return 'error'

        return 'ok'

class PytestRunner(Runner):
    _test_fail_states = ['failed', 'error', None, '']

    def get_test_id(self, test):
        return test.nodeid #.replace('/', '.')

    def init_tests(self):
        logger.debug('Running pytest --collect-only')

        pytest.main(['-p', 'no:terminal', '--collect-only', self.path],
            plugins=[PytestPlugin(runner=self)])

    @classmethod
    def process_init_tests(cls, path, write_pipe, pipe_size, pipe_semaphore):
        logging_tools.configure('pytui-runner.log')

        """ Class method for running in separate process """
        logger.debug('Inside the runner process %s %s %s' % (cls, path, write_pipe))
        runner = cls(path, write_pipe=write_pipe, pipe_size=pipe_size, pipe_semaphore=pipe_semaphore)
        runner.init_tests()
        logger.debug('Inside the runner process end')

    @classmethod
    def process_run_tests(cls, path, failed_only, filtered, write_pipe,
                          pipe_size, pipe_semaphore, filter_value):
        logging_tools.configure('pytui-runner.log')
        runner = cls(path, write_pipe=write_pipe, pipe_size=pipe_size,
                     pipe_semaphore=pipe_semaphore)
        runner.run_tests(failed_only, filter_value)

    def item_collected(self, item):
        # self.tests[self.get_test_id(item)] = item
        self.pipe_send('item_collected', item_id=self.get_test_id(item))

    def run_tests(self, failed_only, filter_value):
        args = ['-p', 'no:terminal', self.path]
        if failed_only:
            args.append('--lf')
        pytest.main(args,
            plugins=[PytestPlugin(runner=self, filter_value=filter_value)])

    def result_state(self, report):
        if not report:
            return ''
        elif report.outcome == 'passed':
            return 'ok'
        elif report.outcome == 'failed':
            return 'failed'
        elif report.outcome == 'skipped':
            return 'skipped'

        logger.warn('Unknown report outcome %s', report.outcome)
        return 'N/A'

<commit_msg>Capture the subprocess' stdout and stderr<commit_after>#!/usr/bin/env python
# encoding: utf-8

import re
import os
import sys
import json
import pytest
import logging
import traceback
import logging_tools
from collections import OrderedDict
import unittest
from StringIO import StringIO
import __builtin__

from plugin import PytestPlugin

logger = logging_tools.get_logger(__name__)

class RollbackImporter:
    def __init__(self):
        "Creates an instance and installs as the global importer"
        self.previousModules = sys.modules.copy()
        self.realImport = __builtin__.__import__
        __builtin__.__import__ = self._import
        self.newModules = {}

    def _import(self, name, globals=None, locals=None, fromlist=[], *args, **kwargs):
        # logger.debug('args: %s', args)
        # logger.debug('kwargs: %s', kwargs)
        result = apply(self.realImport, (name, globals, locals, fromlist))
        self.newModules[name] = 1
        return result

    def uninstall(self):
        logger.debug('uninstalling modules')
        for modname in self.newModules.keys():
            logger.debug('modname: %s', modname)
            if not self.previousModules.has_key(modname):
                # Force reload when modname next imported
                del(sys.modules[modname])
        __builtin__.__import__ = self.realImport

class Runner(object):
    def __init__(self, path='.', write_pipe=None, pipe_size=None, pipe_semaphore=None):
        self.ui = None
        self.path = path
        self.tests = OrderedDict()
        logger.debug('%s Init', self.__class__.__name__)
        self.write_pipe = os.fdopen(write_pipe, 'w', 0)
        self.pipe_size = pipe_size
        self.pipe_semaphore = pipe_semaphore

    def pipe_send(self, method, **kwargs):
        data = '%s\n' % json.dumps({
                'method': method,
                'params': kwargs
        })
        pipe_lock = self.pipe_size.get_lock()
        data_size = len(data)
        logger.debug('writing to pipe size: %s, pipe_size: %s',
                     data_size, self.pipe_size.value)

        if self.pipe_size.value + data_size >= 4096:
            logger.debug('waiting for reader')
            self.pipe_semaphore.clear()
            self.pipe_semaphore.wait()
            logger.debug('reader finished')

        with pipe_lock:
            self.pipe_size.value += data_size
            self.write_pipe.write(data)

    def set_test_result(self, test_id, report, output):
        self.pipe_send('set_test_result',
            test_id=test_id,
            output=output,
            result_state=self.result_state(report),
            when=report.when,
            outcome=report.outcome
        )

    def set_test_state(self, test_id, state):
        self.pipe_send('set_test_state',
            test_id=test_id,
            state=state
        )

    def set_exception_info(self, test_id, excinfo, when):
        exc_type, exc_value, exc_traceback = excinfo._excinfo
        extracted_traceback = traceback.extract_tb(exc_traceback)
        self.pipe_send('set_exception_info',
            test_id=test_id,
            exc_type='TODO',
            exc_value=traceback.format_exception_only(exc_type, exc_value)[-1],
            extracted_traceback=extracted_traceback,
            result='failed',
            when=when
        )

    def get_test_id(self, test):
        raise NotImplementedError()

class UnittestRunner(Runner):
    def get_suite_tests(suite):
        test_list = {}
        for item in suite:
            if isinstance(item, unittest.suite.TestSuite):
                test_list.update(self.get_suite_tests(item))
            else:
                test_list[item.id()] = item

        return OrderedDict(sorted(test_list.iteritems()))

    def get_test_id(self, test):
        return test.id()

    def init_tests(self):
        loader = unittest.TestLoader()
        top_suite = loader.discover(self.path)
        self.tests = self.get_suite_tests(top_suite)

    def reload_tests(self):
        if self.rollbackImporter:
            self.rollbackImporter.uninstall()
        self.rollbackImporter = RollbackImporter()
        self.init_tests()

    def run_tests(self, failed_only=True, filtered=True):
        self.reload_tests()
        tests = self._get_tests(failed_only, filtered)

        for test_id, suite in tests.iteritems():
            self._run_test(test_id)

    def result_state(self, test_result):
        if not test_result:
            return ''
        elif test_result.skipped:
            return 'skipped'
        elif test_result.failures:
            return 'failed'
        elif test_result.errors:
            return 'error'

        return 'ok'

class PytestRunner(Runner):
    _test_fail_states = ['failed', 'error', None, '']

    def get_test_id(self, test):
        return test.nodeid #.replace('/', '.')

    def init_tests(self):
        logger.debug('Running pytest --collect-only')

        pytest.main(['-p', 'no:terminal', '--collect-only', self.path],
            plugins=[PytestPlugin(runner=self)])

    @classmethod
    def process_init_tests(cls, path, write_pipe, pipe_size, pipe_semaphore):
        sys.stdout = StringIO()
        sys.stderr = StringIO()
        logging_tools.configure('pytui-runner.log')

        """ Class method for running in separate process """
        logger.debug('Inside the runner process %s %s %s' % (cls, path, write_pipe))
        runner = cls(path, write_pipe=write_pipe, pipe_size=pipe_size, pipe_semaphore=pipe_semaphore)
        runner.init_tests()
        logger.debug('Inside the runner process end')

    @classmethod
    def process_run_tests(cls, path, failed_only, filtered, write_pipe,
                          pipe_size, pipe_semaphore, filter_value):
        sys.stdout = StringIO()
        sys.stderr = StringIO()
        logging_tools.configure('pytui-runner.log')
        runner = cls(path, write_pipe=write_pipe, pipe_size=pipe_size,
                     pipe_semaphore=pipe_semaphore)
        runner.run_tests(failed_only, filter_value)

    def item_collected(self, item):
        # self.tests[self.get_test_id(item)] = item
        self.pipe_send('item_collected', item_id=self.get_test_id(item))

    def run_tests(self, failed_only, filter_value):
        args = ['-p', 'no:terminal', self.path]
        if failed_only:
            args.append('--lf')
        pytest.main(args,
            plugins=[PytestPlugin(runner=self, filter_value=filter_value)])

    def result_state(self, report):
        if not report:
            return ''
        elif report.outcome == 'passed':
            return 'ok'
        elif report.outcome == 'failed':
            return 'failed'
        elif report.outcome == 'skipped':
            return 'skipped'

        logger.warn('Unknown report outcome %s', report.outcome)
        return 'N/A'

<|endoftext|>


<commit_before># !/usr/bin/python
# coding: utf_8

# Copyright 2016 Stefano Fogarollo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Parse anything there is on the Internet. """

from bs4 import BeautifulSoup

class HtmlTable(str):
    def __init__(self, html_source):
        """
        :param html_source: string
            Html source of table
        """

        str.__init__(html_source)
        self.source = html_source
        self.soup = BeautifulSoup(html_source)

    def parse(self):
        """
        :return: list of list
            List of list of values in table
        """

        data = []  # add name of section
        for row in self.soup.find_all("tr"):  # cycle through all rows
            data_row = []
            for column_label in row.find_all("th"):  # cycle through all labels
                data_row.append(
                    strip_raw_html_string(column_label.text)
                )

            for column in row.find_all("td"):  # cycle through all columns
                data_row.append(
                    strip_raw_html_string(column.text)
                )

            data.append(data_row)
        return data

def html_stripper(string):
    """
    :param string: string
        String to parse
    :return: string
        Given string with raw HTML elements removed
    """

    def is_string_well_formatted(string):
        """
        :param string: string
            String to parse
        :return: bool
            True iff string is good formatted
        """

        # False iff there are at least \n, \r, \t,"  "
        is_bad_formatted = string.find("\n") > 0 or \
                           string.find("\r") > 0 or \
                           string.find("\t") > 0 or \
                           string.find("\\n") > 0 or \
                           string.find("\\r") > 0 or \
                           string.find("\\t") > 0 or \
                           string.find("  ") > 0
        return not is_bad_formatted

    out = string
    while not is_string_well_formatted(out):  # while there are some edits to do
        out = out.replace("\\n", "") \
            .replace("\\r", "") \
            .replace("\\t", "") \
            .replace("\n", "") \
            .replace("\r", "") \
            .replace("\t", "") \
            .replace("  ", " ") \
            .strip()

    return out.encode("utf-8")
<commit_msg>improved html stripper<commit_after># !/usr/bin/python
# coding: utf_8

# Copyright 2016 Stefano Fogarollo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Parse anything there is on the Internet. """

from bs4 import BeautifulSoup

class HtmlTable(str):
    def __init__(self, html_source):
        """
        :param html_source: string
            Html source of table
        """

        str.__init__(html_source)
        self.source = html_source
        self.soup = BeautifulSoup(html_source)

    def parse(self):
        """
        :return: list of list
            List of list of values in table
        """

        data = []  # add name of section
        for row in self.soup.find_all("tr"):  # cycle through all rows
            data_row = []
            for column_label in row.find_all("th"):  # cycle through all labels
                data_row.append(
                    strip_raw_html_string(column_label.text)
                )

            for column in row.find_all("td"):  # cycle through all columns
                data_row.append(
                    strip_raw_html_string(column.text)
                )

            data.append(data_row)
        return data

def html_stripper(string):
    """
    :param string: string
        String to parse
    :return: string
        Given string with raw HTML elements removed
    """

    def is_string_well_formatted(string):
        """
        :param string: string
            String to parse
        :return: bool
            True iff string is good formatted
        """

        # False iff there are at least \n, \r, \t,"  "
        is_bad_formatted = string.find("\n") > 0 or \
                           string.find("\r") > 0 or \
                           string.find("\t") > 0 or \
                           string.find("\\n") > 0 or \
                           string.find("\\r") > 0 or \
                           string.find("\\t") > 0 or \
                           string.find("  ") > 0
        return not is_bad_formatted

    out = string
    while not is_string_well_formatted(out):  # while there are some improvements to do
        out = out.replace("\\n", "") \
            .replace("\\r", "") \
            .replace("\\t", "") \
            .replace("\n", "") \
            .replace("\r", "") \
            .replace("\t", "") \
            .replace("  ", " ") \
            .strip()

    return out.encode("utf-8")
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio
# 2017-04-13

import tornado
import time
import hashlib
from app.Session import Session
from random import Random

class BaseHandler(tornado.web.RequestHandler):

    # 初始化函数
    def initialize(self):
        # 当前请求时间
        self.time = int(time.time())
        # Session
        self.init_session()
        # Version
        self.app_version = self.application.__version__
        # Current Route
        self.url = self.get_current_route()

    # 后面的方法如果重写on_finish方法，需要调用_on_finish
    def _on_finish(self):
        # 更新Session
        self.session.save()
        # 请求逻辑处理结束时关闭数据库连接，如果不关闭可能会造成MySQL Server has gone away 2006错误
        #self.db.close()

    # 重载on_finish
    def on_finish(self):
        self._on_finish()

    # 重载write_error方法
    def write_error(self, status_code, **kwargs):
        title = "%s - %s" % (status_code, self._reason)
        if status_code == 404: # 捕获404
            self.render('page/error.html',title=title)
        elif status_code == 500: # 500可以正常捕获，404好像不行
            #print self.settings.get("serve_traceback")
            msg = ''
            if 'exc_info' in kwargs:
                for i in kwargs['exc_info']:
                    #print type(i)
                    msg += "
%s
" % str(i)
            self.render('page/error.html', title=title, code=status_code, msg=msg)
        else:
            self.render('page/error.html', title=title, code=status_code, msg=status_code)

    # Log Instance
    @property
    def log(self):
        return self.application.log

    # 获取当前路由
    def get_current_route(self):
        uri = self.request.uri.split('?')
        return uri[0]

    # 数据库
    @property
    def db(self):
        return self.application.db

    # Redis
    @property
    def redis(self):
        return self.application.redis

    # 返回Json
    def jsonReturn(self,data):
        self.set_header('Content-Type', 'application/json')
        self.write(data)

    # 格式化时间戳
    def format_time(self,timstamp=None,format='%Y-%m-%d %H:%M:%S'):
        return time.strftime(format, time.localtime(timstamp))

    # 获取当前登录用户
    def get_current_user(self):
        if not self.session.isGuest and self.session.data:
            return self.session.data
        else:
            return None

    # Session初始化
    def init_session(self):
        prefix = self.settings.get('session_prefix')
        expires = self.settings.get('session_expires')
        self.cookie_name = self.settings.get('cookie_name')
        self.sid = self.get_secure_cookie(self.cookie_name)
        self.session = Session(prefix, self.sid, expires, self.redis)

    # MD5计算
    def md5(self,text):
        s = hashlib.md5()
        s.update(text)
        return s.hexdigest()
<commit_msg>Delete md5 from BaseHandler<commit_after>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio
# 2017-04-13

import tornado
import time
import hashlib
from app.Session import Session
from random import Random

class BaseHandler(tornado.web.RequestHandler):

    # 初始化函数
    def initialize(self):
        # 当前请求时间
        self.time = int(time.time())
        # Session
        self.init_session()
        # Version
        self.app_version = self.application.__version__
        # Current Route
        self.url = self.get_current_route()

    # 后面的方法如果重写on_finish方法，需要调用_on_finish
    def _on_finish(self):
        # 更新Session
        self.session.save()
        # 请求逻辑处理结束时关闭数据库连接，如果不关闭可能会造成MySQL Server has gone away 2006错误
        #self.db.close()

    # 重载on_finish
    def on_finish(self):
        self._on_finish()

    # 重载write_error方法
    def write_error(self, status_code, **kwargs):
        title = "%s - %s" % (status_code, self._reason)
        if status_code == 404: # 捕获404
            self.render('page/error.html',title=title)
        elif status_code == 500: # 500可以正常捕获，404好像不行
            #print self.settings.get("serve_traceback")
            msg = ''
            if 'exc_info' in kwargs:
                for i in kwargs['exc_info']:
                    #print type(i)
                    msg += "
%s
" % str(i)
            self.render('page/error.html', title=title, code=status_code, msg=msg)
        else:
            self.render('page/error.html', title=title, code=status_code, msg=status_code)

    # Log Instance
    @property
    def log(self):
        return self.application.log

    # 获取当前路由
    def get_current_route(self):
        uri = self.request.uri.split('?')
        return uri[0]

    # 数据库
    @property
    def db(self):
        return self.application.db

    # Redis
    @property
    def redis(self):
        return self.application.redis

    # 返回Json
    def jsonReturn(self,data):
        self.set_header('Content-Type', 'application/json')
        self.write(data)

    # 格式化时间戳
    def format_time(self,timstamp=None,format='%Y-%m-%d %H:%M:%S'):
        return time.strftime(format, time.localtime(timstamp))

    # 获取当前登录用户
    def get_current_user(self):
        if not self.session.isGuest and self.session.data:
            return self.session.data
        else:
            return None

    # Session初始化
    def init_session(self):
        prefix = self.settings.get('session_prefix')
        expires = self.settings.get('session_expires')
        self.cookie_name = self.settings.get('cookie_name')
        self.sid = self.get_secure_cookie(self.cookie_name)
        self.session = Session(prefix, self.sid, expires, self.redis)
<|endoftext|>


<commit_before>"""
Generate a colorscheme using imagemagick.
"""
import os
import re
import shutil
import subprocess
import sys

from .settings import CACHE_DIR, COLOR_COUNT, __version__
from . import util

def imagemagick(color_count, img, magick_command):
    """Call Imagemagick to generate a scheme."""
    flags = ["-resize", "25%", "-colors", str(color_count),
             "-unique-colors", "txt:-"]

    return subprocess.check_output([*magick_command, img, *flags]).splitlines()

def has_im():
    """Check to see if the user has im installed."""
    if shutil.which("magick"):
        return ["magick", "convert"]

    elif shutil.which("convert"):
        return ["convert"]

    print("error: imagemagick not found, exiting...\n"
          "error: wal requires imagemagick to function.")
    sys.exit(1)

def gen_colors(img, color_count):
    """Format the output from imagemagick into a list
       of hex colors."""
    magick_command = has_im()

    for i in range(0, 20, 1):
        raw_colors = imagemagick(color_count + i, img, magick_command)

        if len(raw_colors) > 16:
            break

        elif i == 19:
            print("colors: Imagemagick couldn't generate a suitable scheme",
                  "for the image. Exiting...")
            sys.exit(1)

        else:
            print("colors: Imagemagick couldn't generate a %s color palette, "
                  "trying a larger palette size %s."
                  % (color_count, color_count + i))

    return [re.search("#.{6}", str(col)).group(0) for col in raw_colors[1:]]

def create_palette(img, colors, light):
    """Sort the generated colors and store them in a dict that
       we will later save in json format."""
    raw_colors = colors[:1] + colors[8:16] + colors[8:-1]

    if light:
        # Manually adjust colors.
        raw_colors[7] = raw_colors[0]
        raw_colors[0] = util.lighten_color(raw_colors[15], 0.9)
        raw_colors[15] = raw_colors[7]
        raw_colors[8] = util.lighten_color(raw_colors[7], 0.25)

        colors = {"wallpaper": img, "alpha": util.Color.alpha_num,
                  "special": {}, "colors": {}}
        colors["special"]["background"] = raw_colors[0]
        colors["special"]["foreground"] = raw_colors[15]
        colors["special"]["cursor"] = raw_colors[15]

        for i, color in enumerate(raw_colors):
            color = util.darken_color(color, 0.30)
            colors["colors"]["color%s" % i] = util.saturate_color(color, 50)

        colors["colors"]["color0"] = raw_colors[0]

    else:
        # Darken the background color slightly.
        if raw_colors[0][1] != "0":
            raw_colors[0] = util.darken_color(raw_colors[0], 0.25)

        # Manually adjust colors.
        raw_colors[7] = util.blend_color(raw_colors[7], "#EEEEEE")
        raw_colors[8] = util.darken_color(raw_colors[7], 0.30)
        raw_colors[15] = util.blend_color(raw_colors[15], "#EEEEEE")

        colors = {"wallpaper": img, "alpha": util.Color.alpha_num,
                  "special": {}, "colors": {}}
        colors["special"]["background"] = raw_colors[0]
        colors["special"]["foreground"] = raw_colors[15]
        colors["special"]["cursor"] = raw_colors[15]

        for i, color in enumerate(raw_colors):
            colors["colors"]["color%s" % i] = color

    return colors

def get(img, cache_dir=CACHE_DIR,
        color_count=COLOR_COUNT, light=False, notify=False):
    """Get the colorscheme."""
    # home_dylan_img_jpg_1.2.2.json
    cache_file = re.sub("[/|\\|.]", "_", img)
    cache_file = os.path.join(cache_dir, "schemes", cache_file + "_" +
                              __version__ + ".json")

    if os.path.isfile(cache_file):
        colors = file(cache_file)
        util.Color.alpha_num = colors["alpha"]
        print("colors: Found cached colorscheme.")

    else:
        util.msg("wal: Generating a colorscheme...", notify)

        colors = gen_colors(img, color_count)
        colors = create_palette(img, colors, light)

        util.save_file_json(colors, cache_file)
        util.msg("wal: Generation complete.", notify)

    return colors

def file(input_file):
    """Import colorscheme from json file."""
    data = util.read_file_json(input_file)

    if "wallpaper" not in data:
        data["wallpaper"] = "None"

    if "alpha" not in data:
        data["alpha"] = "100"

    return data
<commit_msg>colors: Add light theme support.<commit_after>"""
Generate a colorscheme using imagemagick.
"""
import os
import re
import shutil
import subprocess
import sys

from .settings import CACHE_DIR, COLOR_COUNT, __version__
from . import util

def imagemagick(color_count, img, magick_command):
    """Call Imagemagick to generate a scheme."""
    flags = ["-resize", "25%", "-colors", str(color_count),
             "-unique-colors", "txt:-"]

    return subprocess.check_output([*magick_command, img, *flags]).splitlines()

def has_im():
    """Check to see if the user has im installed."""
    if shutil.which("magick"):
        return ["magick", "convert"]

    elif shutil.which("convert"):
        return ["convert"]

    print("error: imagemagick not found, exiting...\n"
          "error: wal requires imagemagick to function.")
    sys.exit(1)

def gen_colors(img, color_count):
    """Format the output from imagemagick into a list
       of hex colors."""
    magick_command = has_im()

    for i in range(0, 20, 1):
        raw_colors = imagemagick(color_count + i, img, magick_command)

        if len(raw_colors) > 16:
            break

        elif i == 19:
            print("colors: Imagemagick couldn't generate a suitable scheme",
                  "for the image. Exiting...")
            sys.exit(1)

        else:
            print("colors: Imagemagick couldn't generate a %s color palette, "
                  "trying a larger palette size %s."
                  % (color_count, color_count + i))

    return [re.search("#.{6}", str(col)).group(0) for col in raw_colors[1:]]

def create_palette(img, colors, light):
    """Sort the generated colors and store them in a dict that
       we will later save in json format."""
    raw_colors = colors[:1] + colors[8:16] + colors[8:-1]

    if light:
        # Manually adjust colors.
        raw_colors[7] = raw_colors[0]
        raw_colors[0] = util.lighten_color(raw_colors[15], 0.9)
        raw_colors[15] = raw_colors[7]
        raw_colors[8] = util.lighten_color(raw_colors[7], 0.25)

    else:
        # Darken the background color slightly.
        if raw_colors[0][1] != "0":
            raw_colors[0] = util.darken_color(raw_colors[0], 0.25)

        # Manually adjust colors.
        raw_colors[7] = util.blend_color(raw_colors[7], "#EEEEEE")
        raw_colors[8] = util.darken_color(raw_colors[7], 0.30)
        raw_colors[15] = util.blend_color(raw_colors[15], "#EEEEEE")

    colors = {"wallpaper": img, "alpha": util.Color.alpha_num,
              "special": {}, "colors": {}}
    colors["special"]["background"] = raw_colors[0]
    colors["special"]["foreground"] = raw_colors[15]
    colors["special"]["cursor"] = raw_colors[15]

    if light:
        for i, color in enumerate(raw_colors):
            color = util.darken_color(color, 0.30)
            colors["colors"]["color%s" % i] = util.saturate_color(color, 50)

        colors["colors"]["color0"] = raw_colors[0]

    else:
        for i, color in enumerate(raw_colors):
            colors["colors"]["color%s" % i] = color

    return colors

def get(img, cache_dir=CACHE_DIR,
        color_count=COLOR_COUNT, light=False, notify=False):
    """Get the colorscheme."""
    # home_dylan_img_jpg_1.2.2.json
    cache_file = re.sub("[/|\\|.]", "_", img)
    cache_file = os.path.join(cache_dir, "schemes", cache_file + "_" +
                              __version__ + ".json")

    if os.path.isfile(cache_file):
        colors = file(cache_file)
        util.Color.alpha_num = colors["alpha"]
        print("colors: Found cached colorscheme.")

    else:
        util.msg("wal: Generating a colorscheme...", notify)

        colors = gen_colors(img, color_count)
        colors = create_palette(img, colors, light)

        util.save_file_json(colors, cache_file)
        util.msg("wal: Generation complete.", notify)

    return colors

def file(input_file):
    """Import colorscheme from json file."""
    data = util.read_file_json(input_file)

    if "wallpaper" not in data:
        data["wallpaper"] = "None"

    if "alpha" not in data:
        data["alpha"] = "100"

    return data
<|endoftext|>