<commit_before># Copyright 2009, BlueDynamics Alliance - http://bluedynamics.com
# GNU General Public License Version 2 or later

import uuid
import inspect
from zodict import zodict
from zope.interface import implements
from zope.location import LocationIterator
from interfaces import INode
from deco import accepts, returns

class Node(zodict):
    """Base node implementation.
    """
    implements(INode)

    def __init__(self, name=None):
        zodict.__init__(self)
        self.__parent__ = None
        self.__name__ = name
        self._index = dict()
        self._uuid = None
        self.uuid = uuid.uuid4()

    def __setitem__(self, key, val):
        if val.uuid in self._index.keys():
            raise ValueError(u"Node with uuid already exists")
        if inspect.isclass(val):
            raise ValueError(u"It isn't allowed to use classes as values.")
        val.__name__ = key
        val.__parent__ = self
        val._index = self._index
        self._index[val.uuid] = val
        zodict.__setitem__(self, key, val)

    def __delitem__(self, key):
        todelete = self[key]
        childkeys = todelete.keys()
        if childkeys:
            for childkey in childkeys:
                del todelete[childkey]
        del self._index[todelete.uuid]
        zodict.__delitem__(self, key)

    @accepts(object, uuid.UUID)
    def set_uuid(self, uuid):
        if uuid in self._index.keys() and self._index[uuid] is not self:
            raise ValueError(u"Given uuid was already used for another Node")
        if self._uuid in self._index.keys():
            del self._index[self._uuid]
        self._index[uuid] = self
        self._uuid = uuid

    @returns(uuid.UUID)
    def get_uuid(self):
        return self._uuid

    uuid = property(get_uuid, set_uuid)

    @property
    def path(self):
        path = list()
        for parent in LocationIterator(self):
            path.append(parent.__name__)
        path.reverse()
        return path

    @property
    def root(self):
        root = None
        for parent in LocationIterator(self):
            root = parent
        return root

    def node(self, uuid):
        return self._index.get(uuid)

    def filtereditems(self, interface):
        for node in self.values():
            if interface.providedBy(node):
                yield node

    def printtree(self):
        print str(self.__class__) + ': ' + str(self.__name__)
        self._printtree(2)

    def _printtree(self, indent):
        for node in self.values():
            name = str(node.__name__)
            print indent * ' ' + str(node.__class__) + ': ' + \
                name[name.find(':') + 1:]
            node._printtree(indent + 2)

    def __repr__(self):
        return '<%s object \'%s\' at %s>' % (self.__class__.__name__,
                                             str(self.__name__),
                                             hex(id(self))[:-1])

    __str__ = __repr__
<commit_msg>simplified printtree and made it more flexible<commit_after># Copyright 2009, BlueDynamics Alliance - http://bluedynamics.com
# GNU General Public License Version 2 or later

import uuid
import inspect
from zodict import zodict
from zope.interface import implements
from zope.location import LocationIterator
from interfaces import INode
from deco import accepts, returns

class Node(zodict):
    """Base node implementation.
    """
    implements(INode)

    def __init__(self, name=None):
        zodict.__init__(self)
        self.__parent__ = None
        self.__name__ = name
        self._index = dict()
        self._uuid = None
        self.uuid = uuid.uuid4()

    def __setitem__(self, key, val):
        if val.uuid in self._index.keys():
            raise ValueError(u"Node with uuid already exists")
        if inspect.isclass(val):
            raise ValueError(u"It isn't allowed to use classes as values.")
        val.__name__ = key
        val.__parent__ = self
        val._index = self._index
        self._index[val.uuid] = val
        zodict.__setitem__(self, key, val)

    def __delitem__(self, key):
        todelete = self[key]
        childkeys = todelete.keys()
        if childkeys:
            for childkey in childkeys:
                del todelete[childkey]
        del self._index[todelete.uuid]
        zodict.__delitem__(self, key)

    @accepts(object, uuid.UUID)
    def set_uuid(self, uuid):
        if uuid in self._index.keys() and self._index[uuid] is not self:
            raise ValueError(u"Given uuid was already used for another Node")
        if self._uuid in self._index.keys():
            del self._index[self._uuid]
        self._index[uuid] = self
        self._uuid = uuid

    @returns(uuid.UUID)
    def get_uuid(self):
        return self._uuid

    uuid = property(get_uuid, set_uuid)

    @property
    def path(self):
        path = list()
        for parent in LocationIterator(self):
            path.append(parent.__name__)
        path.reverse()
        return path

    @property
    def root(self):
        root = None
        for parent in LocationIterator(self):
            root = parent
        return root

    def node(self, uuid):
        return self._index.get(uuid)

    def filtereditems(self, interface):
        for node in self.values():
            if interface.providedBy(node):
                yield node

    @property
    def noderepr(self):
        name = str(self.__name__)
        return str(self.__class__) + ': ' + name[name.find(':') + 1:]

    def printtree(self, indent=0):
        print "%s%s" % (indent * ' ', self.noderepr)
        for node in self.values():
            node.printtree(indent+2)

    def __repr__(self):
        return "<%s object '%s' at %s>" % (self.__class__.__name__,
                                           str(self.__name__),
                                           hex(id(self))[:-1])

    __str__ = __repr__
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Communicate with the smart electricity meter [KAMSTRUP].

Store data from a Kamstrup smart-electricity meter in a sqlite3 database.
"""

import configparser
import datetime as dt
import os
import re
import sqlite3
import sys
import syslog
import time
import traceback

import mausy5043funcs.fileops3 as mf
import serial
# noinspection PyUnresolvedReferences
from mausy5043libs.libdaemon3 import Daemon

# constants
DEBUG = False
IS_JOURNALD = os.path.isfile('/bin/journalctl')
MYID = "".join(list(filter(str.isdigit, os.path.realpath(__file__).split('/')[-1])))
MYAPP = os.path.realpath(__file__).split('/')[-3]
NODE = os.uname()[1]

# noinspection PyUnresolvedReferences
class MyDaemon(Daemon):
  """Override Daemon-class run() function."""

  # pylint: disable=too-few-public-methods

  # noinspection PyUnresolvedReferences
  @staticmethod
  def run():
    """Execute main loop."""
    iniconf = configparser.ConfigParser()
    iniconf.read(f"{os.environ['HOME']}/{MYAPP}/config.ini")
    report_time = iniconf.getint(MYID, "reporttime")
    fdatabase = f"{os.environ['HOME']}/{iniconf.get(MYID, 'databasefile')}"
    sqlcmd = iniconf.get(MYID, 'sqlcmd')
    samples_averaged = iniconf.getint(MYID, 'samplespercycle') * iniconf.getint(MYID, 'cycles')
    sample_time = report_time / iniconf.getint(MYID, 'samplespercycle')
    data = []

    test_db_connection(fdatabase)

    port.open()
    # noinspection PyStatementEffect
    serial.XON  # pylint: disable=W0104
    while True:
      try:
        start_time = time.time()
        result = do_work()
        result = result.split(',')
        mf.syslog_trace(f"Result   : {result}", False, DEBUG)
        # data.append(list(map(int, result)))
        data.append([int(d) for d in result])
        if len(data) > samples_averaged:
          data.pop(0)
        # mf.syslog_trace(f"Data     : {data}", False, DEBUG)

        # report sample average
        if start_time % report_time < sample_time:
          # somma       = list(map(sum, zip(*data)))
          somma = [sum(d) for d in zip(*data)]
          # not all entries should be float
          # ['3088596', '3030401', '270', '0', '0', '0', '1', '1']
          # averages    = [format(sm / len(data), '.2f') for sm in somma]
          averages = data[-1]
          averages[2] = int(somma[2] / len(data))  # avg powerin
          averages[5] = int(somma[5] / len(data))  # avg powerout
          mf.syslog_trace(f"Averages : {averages}", False, DEBUG)
          if averages[0] > 0:
            do_add_to_database(averages, fdatabase, sqlcmd)

        pause_time = (sample_time
                      - (time.time() - start_time)
                      - (start_time % sample_time))
        if pause_time > 0:
          mf.syslog_trace(f"Waiting  : {pause_time}s", False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
          # no need to wait for the next cycles
          # the meter will pace the meaurements
          # any required waiting will be inside gettelegram()
          # time.sleep(pause_time)
        else:
          mf.syslog_trace(f"Behind   : {pause_time}s", False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
      except Exception:
        mf.syslog_trace("Unexpected error in run()", syslog.LOG_CRIT, DEBUG)
        mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
        raise

def do_work():
  """Push the results out to a file."""
  electra1in = 0
  electra2in = 0
  powerin = 0
  electra1out = 0
  electra2out = 0
  powerout = 0
  tarif = 0
  swits = 1

  telegram, status = gettelegram()

  if status == 1:
    for element in telegram:
      line = re.split(r'[\(\*\)]', element)
      # ['1-0:1.8.1', '00175.402', 'kWh', '']  T1 in
      if line[0] == '1-0:1.8.1':
        electra1in = int(float(line[1]) * 1000)
      # ['1-0:1.8.2', '00136.043', 'kWh', '']  T2 in
      if line[0] == '1-0:1.8.2':
        electra2in = int(float(line[1]) * 1000)
      # ['1-0:2.8.1', '00000.000', 'kWh', '']  T1 out
      if line[0] == '1-0:2.8.1':
        electra1out = int(float(line[1]) * 1000)
      # ['1-0:2.8.2', '00000.000', 'kWh', '']  T2 out
      if line[0] == '1-0:2.8.2':
        electra2out = int(float(line[1]) * 1000)
      # ['0-0:96.14.0', '0002', '']  tarif 1 or 2
      if line[0] == '0-0:96.14.0':
        tarif = int(line[1])
      # ['1-0:1.7.0', '0000.32', 'kW', '']  power in
      if line[0] == '1-0:1.7.0':
        powerin = int(float(line[1]) * 1000)
      # ['1-0:2.7.0', '0000.00', 'kW', ''] power out
      if line[0] == '1-0:2.7.0':
        powerout = int(float(line[1]) * 1000)
      # ['0-0:17.0.0', '999', 'A', ''] unknown; not recorded
      # ['0-0:96.3.10', '1', '']  powerusage (1) or powermanufacturing ()
      if line[0] == '0-0:96.3.10':
        swits = int(line[1])
      # ['0-0:96.13.1', '', '']
      # not recorded
      # ['0-0:96.13.0', '', '']
      # not recorded

  return f'{electra1in}, {electra2in}, {powerin}, {electra1out}, {electra2out}, {powerout}, {tarif}, {swits}'

# noinspection PyUnresolvedReferences
def gettelegram():
  """Fetch a telegram from the serialport."""
  # flag used to exit the while-loop
  abort = 0
  # countdown counter used to prevent infinite loops
  loops2go = 40
  # storage space for the telegram
  telegram = []
  # end of line delimiter
  # delim = "\x0a"

  while abort == 0:
    try:
      # line = "".join(iter(lambda: port.read(1), delim)).strip()
      line = str(port.readline().strip(), 'utf-8')
      if line == "!":
        abort = 1
      if line != "":
        telegram.append(line)
    except serial.SerialException:
      mf.syslog_trace("*** Serialport read error:", syslog.LOG_CRIT, DEBUG)
      mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
      abort = 2
    except UnicodeDecodeError:
      mf.syslog_trace("*** Unicode Decode error:", syslog.LOG_CRIT, DEBUG)
      mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
      abort = 2

    loops2go = loops2go - 1
    if loops2go < 0:
      abort = 3

  # test for correct start of telegram
  if telegram[0][0] != "/":
    abort = 2

  if abort == 1:
    with open('/tmp/kamstrup.raw', 'w') as output_file:
      for line in telegram:
        output_file.write(f'{line}\n')

  # Return codes:
  # abort == 1 indicates a successful read
  # abort == 2 means that a serial port read/write error occurred
  # abort == 3 no valid data after several attempts
  return telegram, abort

def do_add_to_database(result, fdatabase, sql_cmd):
  """Commit the results to the database."""
  # Get the time and date in human-readable form and UN*X-epoch...
  conn = None
  cursor = None
  out_date = dt.datetime.now()  # time.strftime('%Y-%m-%dT%H:%M:%S')
  out_epoch = int(out_date.timestamp())
  results = (out_date, out_epoch,
             result[0], result[1], result[2],
             result[3], result[4], result[5],
             result[6], result[7])
  mf.syslog_trace(f"   @: {out_date}", False, DEBUG)
  mf.syslog_trace(f"    : {results}", False, DEBUG)

  err_flag = True
  while err_flag:
    try:
      conn = create_db_connection(fdatabase)
      cursor = conn.cursor()
      cursor.execute(sql_cmd, results)
      cursor.close()
      conn.commit()
      conn.close()
      err_flag = False
    except sqlite3.OperationalError:
      if cursor:
        cursor.close()
      if conn:
        conn.close()

def create_db_connection(database_file):
  """
  Create a database connection to the SQLite3 database specified by database_file.
  """
  consql = None
  mf.syslog_trace(f"Connecting to: {database_file}", False, DEBUG)
  try:
    consql = sqlite3.connect(database_file, timeout=9000)
    # if consql:    # dB initialised succesfully -> get a cursor on the dB and run a test.
    #  cursql = consql.cursor()
    #  cursql.execute("SELECT sqlite_version()")
    #  versql = cursql.fetchone()
    #  cursql.close()
    #  logtext = f"Attached to SQLite3 server : {versql}"
    #  syslog.syslog(syslog.LOG_INFO, logtext)
    return consql
  except sqlite3.Error:
    mf.syslog_trace("Unexpected SQLite3 error when connecting to server.", syslog.LOG_CRIT, DEBUG)
    mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
    if consql:  # attempt to close connection to SQLite3 server
      consql.close()
      mf.syslog_trace(" ** Closed SQLite3 connection. **", syslog.LOG_CRIT, DEBUG)
    raise

def test_db_connection(fdatabase):
  """
  Test & log database engine connection.
  """
  try:
    conn = create_db_connection(fdatabase)
    cursor = conn.cursor()
    cursor.execute("SELECT sqlite_version();")
    versql = cursor.fetchone()
    cursor.close()
    conn.commit()
    conn.close()
    syslog.syslog(syslog.LOG_INFO, f"Attached to SQLite3 server: {versql}")
  except sqlite3.Error:
    mf.syslog_trace("Unexpected SQLite3 error during test.", syslog.LOG_CRIT, DEBUG)
    mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
    raise

if __name__ == "__main__":
  daemon = MyDaemon(f'/tmp/{MYAPP}/{MYID}.pid')  # pylint: disable=C0103

  # noinspection PyUnresolvedReferences
  port = serial.Serial()  # pylint: disable=C0103
  port.baudrate = 9600
  # noinspection PyUnresolvedReferences
  port.bytesize = serial.SEVENBITS
  # noinspection PyUnresolvedReferences
  port.parity = serial.PARITY_EVEN
  # noinspection PyUnresolvedReferences
  port.stopbits = serial.STOPBITS_ONE
  port.xonxoff = 1
  port.rtscts = 0
  port.dsrdtr = 0
  port.timeout = 15
  port.port = '/dev/ttyUSB0'

  # initialise logging
  syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)

  if len(sys.argv) == 2:
    if sys.argv[1] == 'start':
      daemon.start()
    elif sys.argv[1] == 'stop':
      daemon.stop()
    elif sys.argv[1] == 'restart':
      daemon.restart()
    elif sys.argv[1] == 'debug':
      # assist with debugging.
      print("Debug-mode started. Use <Ctrl>+C to stop.")
      DEBUG = True
      mf.syslog_trace("Daemon logging is ON", syslog.LOG_DEBUG, DEBUG)
      daemon.run()
    else:
      print("Unknown command")
      sys.exit(2)
    sys.exit(0)
  else:
    print("usage: {0!s} start|stop|restart|debug".format(sys.argv[0]))
    sys.exit(2)
<commit_msg>suppress warning<commit_after>#!/usr/bin/env python3

"""
Communicate with the smart electricity meter [KAMSTRUP].

Store data from a Kamstrup smart-electricity meter in a sqlite3 database.
"""

import configparser
import datetime as dt
import os
import re
import sqlite3
import sys
import syslog
import time
import traceback

# noinspection PyUnresolvedReferences
import mausy5043funcs.fileops3 as mf
import serial
# noinspection PyUnresolvedReferences
from mausy5043libs.libdaemon3 import Daemon

# constants
DEBUG = False
IS_JOURNALD = os.path.isfile('/bin/journalctl')
MYID = "".join(list(filter(str.isdigit, os.path.realpath(__file__).split('/')[-1])))
MYAPP = os.path.realpath(__file__).split('/')[-3]
NODE = os.uname()[1]

# noinspection PyUnresolvedReferences
class MyDaemon(Daemon):
  """Override Daemon-class run() function."""

  # pylint: disable=too-few-public-methods

  # noinspection PyUnresolvedReferences
  @staticmethod
  def run():
    """Execute main loop."""
    iniconf = configparser.ConfigParser()
    iniconf.read(f"{os.environ['HOME']}/{MYAPP}/config.ini")
    report_time = iniconf.getint(MYID, "reporttime")
    fdatabase = f"{os.environ['HOME']}/{iniconf.get(MYID, 'databasefile')}"
    sqlcmd = iniconf.get(MYID, 'sqlcmd')
    samples_averaged = iniconf.getint(MYID, 'samplespercycle') * iniconf.getint(MYID, 'cycles')
    sample_time = report_time / iniconf.getint(MYID, 'samplespercycle')
    data = []

    test_db_connection(fdatabase)

    port.open()
    # noinspection PyStatementEffect
    serial.XON  # pylint: disable=W0104
    while True:
      try:
        start_time = time.time()
        result = do_work()
        result = result.split(',')
        mf.syslog_trace(f"Result   : {result}", False, DEBUG)
        # data.append(list(map(int, result)))
        data.append([int(d) for d in result])
        if len(data) > samples_averaged:
          data.pop(0)
        # mf.syslog_trace(f"Data     : {data}", False, DEBUG)

        # report sample average
        if start_time % report_time < sample_time:
          # somma       = list(map(sum, zip(*data)))
          somma = [sum(d) for d in zip(*data)]
          # not all entries should be float
          # ['3088596', '3030401', '270', '0', '0', '0', '1', '1']
          # averages    = [format(sm / len(data), '.2f') for sm in somma]
          averages = data[-1]
          averages[2] = int(somma[2] / len(data))  # avg powerin
          averages[5] = int(somma[5] / len(data))  # avg powerout
          mf.syslog_trace(f"Averages : {averages}", False, DEBUG)
          if averages[0] > 0:
            do_add_to_database(averages, fdatabase, sqlcmd)

        pause_time = (sample_time
                      - (time.time() - start_time)
                      - (start_time % sample_time))
        if pause_time > 0:
          mf.syslog_trace(f"Waiting  : {pause_time}s", False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
          # no need to wait for the next cycles
          # the meter will pace the meaurements
          # any required waiting will be inside gettelegram()
          # time.sleep(pause_time)
        else:
          mf.syslog_trace(f"Behind   : {pause_time}s", False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
      except Exception:
        mf.syslog_trace("Unexpected error in run()", syslog.LOG_CRIT, DEBUG)
        mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
        raise

def do_work():
  """Push the results out to a file."""
  electra1in = 0
  electra2in = 0
  powerin = 0
  electra1out = 0
  electra2out = 0
  powerout = 0
  tarif = 0
  swits = 1

  telegram, status = gettelegram()

  if status == 1:
    for element in telegram:
      line = re.split(r'[\(\*\)]', element)
      # ['1-0:1.8.1', '00175.402', 'kWh', '']  T1 in
      if line[0] == '1-0:1.8.1':
        electra1in = int(float(line[1]) * 1000)
      # ['1-0:1.8.2', '00136.043', 'kWh', '']  T2 in
      if line[0] == '1-0:1.8.2':
        electra2in = int(float(line[1]) * 1000)
      # ['1-0:2.8.1', '00000.000', 'kWh', '']  T1 out
      if line[0] == '1-0:2.8.1':
        electra1out = int(float(line[1]) * 1000)
      # ['1-0:2.8.2', '00000.000', 'kWh', '']  T2 out
      if line[0] == '1-0:2.8.2':
        electra2out = int(float(line[1]) * 1000)
      # ['0-0:96.14.0', '0002', '']  tarif 1 or 2
      if line[0] == '0-0:96.14.0':
        tarif = int(line[1])
      # ['1-0:1.7.0', '0000.32', 'kW', '']  power in
      if line[0] == '1-0:1.7.0':
        powerin = int(float(line[1]) * 1000)
      # ['1-0:2.7.0', '0000.00', 'kW', ''] power out
      if line[0] == '1-0:2.7.0':
        powerout = int(float(line[1]) * 1000)
      # ['0-0:17.0.0', '999', 'A', ''] unknown; not recorded
      # ['0-0:96.3.10', '1', '']  powerusage (1) or powermanufacturing ()
      if line[0] == '0-0:96.3.10':
        swits = int(line[1])
      # ['0-0:96.13.1', '', '']
      # not recorded
      # ['0-0:96.13.0', '', '']
      # not recorded

  return f'{electra1in}, {electra2in}, {powerin}, {electra1out}, {electra2out}, {powerout}, {tarif}, {swits}'

# noinspection PyUnresolvedReferences
def gettelegram():
  """Fetch a telegram from the serialport."""
  # flag used to exit the while-loop
  abort = 0
  # countdown counter used to prevent infinite loops
  loops2go = 40
  # storage space for the telegram
  telegram = []
  # end of line delimiter
  # delim = "\x0a"

  while abort == 0:
    try:
      # line = "".join(iter(lambda: port.read(1), delim)).strip()
      line = str(port.readline().strip(), 'utf-8')
      if line == "!":
        abort = 1
      if line != "":
        telegram.append(line)
    except serial.SerialException:
      mf.syslog_trace("*** Serialport read error:", syslog.LOG_CRIT, DEBUG)
      mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
      abort = 2
    except UnicodeDecodeError:
      mf.syslog_trace("*** Unicode Decode error:", syslog.LOG_CRIT, DEBUG)
      mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
      abort = 2

    loops2go = loops2go - 1
    if loops2go < 0:
      abort = 3

  # test for correct start of telegram
  if telegram[0][0] != "/":
    abort = 2

  if abort == 1:
    with open('/tmp/kamstrup.raw', 'w') as output_file:
      for line in telegram:
        output_file.write(f'{line}\n')

  # Return codes:
  # abort == 1 indicates a successful read
  # abort == 2 means that a serial port read/write error occurred
  # abort == 3 no valid data after several attempts
  return telegram, abort

def do_add_to_database(result, fdatabase, sql_cmd):
  """Commit the results to the database."""
  # Get the time and date in human-readable form and UN*X-epoch...
  conn = None
  cursor = None
  out_date = dt.datetime.now()  # time.strftime('%Y-%m-%dT%H:%M:%S')
  out_epoch = int(out_date.timestamp())
  results = (out_date, out_epoch,
             result[0], result[1], result[2],
             result[3], result[4], result[5],
             result[6], result[7])
  mf.syslog_trace(f"   @: {out_date}", False, DEBUG)
  mf.syslog_trace(f"    : {results}", False, DEBUG)

  err_flag = True
  while err_flag:
    try:
      conn = create_db_connection(fdatabase)
      cursor = conn.cursor()
      cursor.execute(sql_cmd, results)
      cursor.close()
      conn.commit()
      conn.close()
      err_flag = False
    except sqlite3.OperationalError:
      if cursor:
        cursor.close()
      if conn:
        conn.close()

def create_db_connection(database_file):
  """
  Create a database connection to the SQLite3 database specified by database_file.
  """
  consql = None
  mf.syslog_trace(f"Connecting to: {database_file}", False, DEBUG)
  try:
    consql = sqlite3.connect(database_file, timeout=9000)
    # if consql:    # dB initialised succesfully -> get a cursor on the dB and run a test.
    #  cursql = consql.cursor()
    #  cursql.execute("SELECT sqlite_version()")
    #  versql = cursql.fetchone()
    #  cursql.close()
    #  logtext = f"Attached to SQLite3 server : {versql}"
    #  syslog.syslog(syslog.LOG_INFO, logtext)
    return consql
  except sqlite3.Error:
    mf.syslog_trace("Unexpected SQLite3 error when connecting to server.", syslog.LOG_CRIT, DEBUG)
    mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
    if consql:  # attempt to close connection to SQLite3 server
      consql.close()
      mf.syslog_trace(" ** Closed SQLite3 connection. **", syslog.LOG_CRIT, DEBUG)
    raise

def test_db_connection(fdatabase):
  """
  Test & log database engine connection.
  """
  try:
    conn = create_db_connection(fdatabase)
    cursor = conn.cursor()
    cursor.execute("SELECT sqlite_version();")
    versql = cursor.fetchone()
    cursor.close()
    conn.commit()
    conn.close()
    syslog.syslog(syslog.LOG_INFO, f"Attached to SQLite3 server: {versql}")
  except sqlite3.Error:
    mf.syslog_trace("Unexpected SQLite3 error during test.", syslog.LOG_CRIT, DEBUG)
    mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
    raise

if __name__ == "__main__":
  daemon = MyDaemon(f'/tmp/{MYAPP}/{MYID}.pid')  # pylint: disable=C0103

  # noinspection PyUnresolvedReferences
  port = serial.Serial()  # pylint: disable=C0103
  port.baudrate = 9600
  # noinspection PyUnresolvedReferences
  port.bytesize = serial.SEVENBITS
  # noinspection PyUnresolvedReferences
  port.parity = serial.PARITY_EVEN
  # noinspection PyUnresolvedReferences
  port.stopbits = serial.STOPBITS_ONE
  port.xonxoff = 1
  port.rtscts = 0
  port.dsrdtr = 0
  port.timeout = 15
  port.port = '/dev/ttyUSB0'

  # initialise logging
  syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)

  if len(sys.argv) == 2:
    if sys.argv[1] == 'start':
      daemon.start()
    elif sys.argv[1] == 'stop':
      daemon.stop()
    elif sys.argv[1] == 'restart':
      daemon.restart()
    elif sys.argv[1] == 'debug':
      # assist with debugging.
      print("Debug-mode started. Use <Ctrl>+C to stop.")
      DEBUG = True
      mf.syslog_trace("Daemon logging is ON", syslog.LOG_DEBUG, DEBUG)
      daemon.run()
    else:
      print("Unknown command")
      sys.exit(2)
    sys.exit(0)
  else:
    print("usage: {0!s} start|stop|restart|debug".format(sys.argv[0]))
    sys.exit(2)
<|endoftext|>


<commit_before># Copyright (c) 2013 Greg Lange
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.

# This file includes code taken from or based on code from:
# https://github.com/openstack/swift
# and
# http://stackoverflow.com/questions/12676393/creating-python-2-7-daemon-with-pep-3143
#
# When the code was taken from swift (and then possibly modified), it's marked
# by the comment "from swift".

from __future__ import with_statement

from ConfigParser import ConfigParser
import errno
import logging
import logging.handlers
from optparse import OptionParser
import pwd
import os
from random import random
import signal
import socket
import sys
import time

# from swift
class LoggerFileObject(object):
    """
    Used to capture stderr/stdout.
    """

    def __init__(self, logger):
        self.logger = logger

    def write(self, value):
        value = value.strip()
        if value:
            if 'Connection reset by peer' in value:
                self.logger.error('STDOUT: Connection reset by peer')
            else:
                self.logger.error('STDOUT: %s', value)

    def writelines(self, values):
        self.logger.error('STDOUT: %s', '#012'.join(values))

    def close(self):
        pass

    def flush(self):
        pass

    def __iter__(self):
        return self

    def next(self):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def read(self, size=-1):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def readline(self, size=-1):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def tell(self):
        return 0

    def xreadlines(self):
        return self

# from swift
def drop_privileges(user):
    """
    Sets the userid/groupid of the current process, get session leader, etc.

    :param user: User name to change privileges to
    """
    user = pwd.getpwnam(user)
    if os.geteuid() == 0:
        os.setgroups([])
    os.setgid(user[3])
    os.setuid(user[2])
    os.environ['HOME'] = user[5]
    try:
        os.setsid()
    except OSError:
        pass
    os.chdir('/')   # in case you need to rmdir on where you started the daemon
    os.umask(0o22)  # ensure files are created with the correct privileges

def get_command_line(command_line, dargs_parser, args_parser):
    """
    Parses the command line.

    Command line should be of the form:
    [common daemon args] command [unique daemon args].

    Returns common daemon args, command, and unique daemon args.
    """
    dargs = dargs_parser.parse_args(command_line)
    command = dargs[1][0]
    if command not in Daemon.commands:
        raise ValueError('Invalid daemon command')
    args = args_parser.parse_args(dargs[1][1:])
    return dargs, command, args

def get_project_from_conf_path(conf_path):
    if not os.path.isfile(conf_path):
        raise ValueError('File path expected')

    conf_file = os.path.basename(conf_path)

    if not conf_file.endswith('.conf'):
        raise ValueError('Conf file should end with .conf')

    return conf_file[:-len('.conf')]

# from swift
def list_from_csv(comma_separated_str):
    """
    Splits the str given and returns a properly stripped list of the comma
    separated values.
    """
    if comma_separated_str:
        return [v.strip() for v in comma_separated_str.split(',') if v.strip()]
    return []

def parse_run_name():
    """
    Returns the parts of the run name of the daemon.

    The run name should be of the form: project-daemon

    This is used to determine the config location/section name.
    """
    command = os.path.split(sys.argv[0])[1]
    parts = command.split('-')
    if len(parts) != 2:
        raise ValueError()
    return parts

# from swift
def read_config(conf_path):
    """
    Reads a config and returns its sections/values.
    """
    c = ConfigParser()
    if not c.read(conf_path):
        print "Unable to read config from %s" % conf_path
        sys.exit(1)
    conf = {}
    for s in c.sections():
        conf.update({s: dict(c.items(s))})
    conf['__file__'] = conf_path
    return conf

class Daemon(object):
    """
    A class for building daemons.

    It takes care of things common to all daemons.
    """

    commands = 'restart run_once start status stop'.split()
    handler4logger = {}

    def __init__(self, global_conf, conf_section, pid_file_path, dargs, args):
        self.global_conf = global_conf
        self.conf_section = conf_section
        self.conf = self.global_conf[conf_section]
        self.pid_file_path = pid_file_path
        self.dargs = dargs
        self.args = args
        self.logger = self.get_logger(self.conf)
        self.user = self.conf['user']
        self.interval = int(self.conf.get('interval', 5))

    # from swift
    def capture_stdio(self):
        return
        """
        Log unhandled exceptions, close stdio, capture stdout and stderr.
        """
        # log uncaught exceptions
        sys.excepthook = lambda * exc_info: \
            self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)

        # collect stdio file desc not in use for logging
        stdio_files = [sys.stdin, sys.stdout, sys.stderr]

        with open(os.devnull, 'r+b') as nullfile:
            # close stdio (excludes fds open for logging)
            for f in stdio_files:
                # some platforms throw an error when attempting an stdin flush
                try:
                    f.flush()
                except IOError:
                    pass

                try:
                    os.dup2(nullfile.fileno(), f.fileno())
                except OSError:
                    pass

        # FUTURE: make the capture optional?
        sys.stdout = LoggerFileObject(self.logger)
        sys.stderr = LoggerFileObject(self.logger)

    def daemonize(self):
        """
        Daemonizes the current process.

        Returns False if the process is not the daemon.

        Returns True if process is the daemon.
        """
        try:
            pid = os.fork()
            if pid > 0:
                return False
        except Exception:
            raise RuntimeError('Fork failed')

        drop_privileges(self.user)

        def kill_children(*args):
            signal.signal(signal.SIGTERM, signal.SIG_IGN)
            os.killpg(0, signal.SIGTERM)
            sys.exit()

        signal.signal(signal.SIGTERM, kill_children)

        # FUTURE: something that logs when the daemon is killed?

        self.capture_stdio()

        with open(self.pid_file_path, 'w+') as fd:
            pid = os.getpid()
            fd.write('%d\n' % pid)

        return True

    @classmethod
    def get_args_parser(cls):
        """
        Override to parse options unique to your daemon.

        Returns an OptionParser.
        """
        return OptionParser()

    @classmethod
    def get_dargs_parser(cls):
        """
        Returns an OptionParse for options common to all daemons.

        Returns an OptionParser.
        """
        # FUTURE: add things that can be overridden on command line
        parser = OptionParser()
        parser.add_option(
            "--eventlet_patch", action="store_false", dest="eventlet_patch",
            default=False, help="add eventlet patch")
        parser.disable_interspersed_args()
        return parser

    # from swift
    @classmethod
    def get_logger(cls, conf):
        """
        Returns a logger configured from the conf.
        """
        if not conf:
            conf = {}
        name = conf.get('log_name', 'daemonx')
        log_route = conf.get('log_route', name)
        logger = logging.getLogger(log_route)
        logger.propagate = False

        # get_logger will only ever add one SysLog Handler to a logger
        if logger in cls.handler4logger:
            logger.removeHandler(cls.handler4logger[logger])

        # facility for this logger will be set by last call wins
        facility = getattr(
            logging.handlers.SysLogHandler,
            conf.get('log_facility', 'LOG_LOCAL0'),
            logging.handlers.SysLogHandler.LOG_LOCAL0)
        udp_host = conf.get('log_udp_host')
        if udp_host:
            udp_port = int(
                conf.get('log_udp_port',
                logging.handlers.SYSLOG_UDP_PORT))
            handler = logging.handlers.SysLogHandler(
                address=(udp_host, udp_port), facility=facility)
        else:
            log_address = conf.get('log_address', '/dev/log')
            try:
                handler = logging.handlers.SysLogHandler(
                    address=log_address, facility=facility)
            except socket.error, e:
                # Either /dev/log isn't a UNIX socket or it does not exist
                # at all
                if e.errno not in [errno.ENOTSOCK, errno.ENOENT]:
                    raise e
                handler = logging.handlers.SysLogHandler(facility=facility)
        logger.addHandler(handler)
        cls.handler4logger[logger] = handler

        # set the level for the logger
        logger.setLevel(
            getattr(
                logging, conf.get('log_level', 'INFO').upper(), logging.INFO))

        return logger

    def get_pid(self):
        """
        Reads and returns the daemon's pid from pid file on disk>

        Returns None on failure.
        """
        try:
            with open(self.pid_file_path, 'r') as fd:
                pid = int(fd.read().strip())
        except IOError:
            pid = None
        return pid

    def run(self):
        """
        Runs the daemon.

        It calls run_forever().
        """
        self.run_forever()

    @classmethod
    def run_command(
            cls, conf_path, conf_section, command_line, project=None,
            daemon_name=None):
        """
        Sends the command specified on the command line to the daemon.
        """
        # read config
        global_conf = read_config(conf_path)

        # get project/daemon name
        if not (project and daemon_name):
            project = get_project_from_conf_path(conf_path)
            daemon_name = conf_section

        # get/import class from config
        import_target, class_name = \
            global_conf[conf_section]['class'].rsplit('.', 1)
        module = __import__(import_target, fromlist=[import_target])
        daemon_class = getattr(module, class_name)

        # parse command line, get command to run on daemon
        dargs_parser = cls.get_dargs_parser()
        args_parser = cls.get_args_parser()
        dargs, command, args = get_command_line(
            command_line, dargs_parser, args_parser)

        # check command
        if command not in cls.commands:
            raise ValueError('Invalid command')

        # get pid file path
        pid_file_path = '/var/run/%s/%s.pid' % (project, daemon_name)

        # create daemon
        daemon = daemon_class(
            global_conf, conf_section, pid_file_path, dargs, args)

        # get and run method for command
        method = getattr(daemon, command)
        try:
            method()
        except Exception, e:
            print e

    @classmethod
    def run_command_from_script(cls):
        """
        Runs the command on the daemon.

        Project and daemon name are determined from the script run.
        """
        project, daemon_name = parse_run_name()
        conf_path = '/etc/%s/%s.conf' % (project, project)
        cls.run_command(
            conf_path, daemon_name, list(sys.argv[1:]), project, daemon_name)

    def run_forever(self):
        """
        Run the daemon forever.

        Sleeps as need be to not run more than once in each interval.

        Calls run_once().
        """
        time.sleep(random() * self.interval)
        while True:
            try:
                self.run_once()
            except Exception:
                self.logger.exception('run_once()')
            time.sleep(self.interval)

    def restart(self):
        """
        Restarts the daemon.
        """
        self.stop()
        self.start()

    def run_once(self):
        """
        Override this to define what the daemon does.
        """
        raise NotImplementedError('run_once not implemented')

    def start(self):
        """
        Starts the daemon.
        """
        pid = self.get_pid()
        if pid:
            raise RuntimeError('Daemon alredy running')

        if self.daemonize():
            self.run()

    def status(self):
        """
        Prints the status of the daemon.
        """
        pid = self.get_pid()
        if pid:
            print 'Daemon is running with pid: %d' % pid
        else:
            print 'Daemon is not running'

    def stop(self):
        """
        Stops the daemon.
        """
        pid = self.get_pid()

        if not pid:
            print 'Daemon does not seem to be running'
            return

        try:
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(.1)
        except OSError, e:
            if str(e).find('No such process') > 0:
                if os.path.exists(self.pid_file_path):
                    os.remove(self.pid_file_path)
                else:
                    raise
<commit_msg>removed a debug return<commit_after># Copyright (c) 2013 Greg Lange
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.

# This file includes code taken from or based on code from:
# https://github.com/openstack/swift
# and
# http://stackoverflow.com/questions/12676393/creating-python-2-7-daemon-with-pep-3143
#
# When the code was taken from swift (and then possibly modified), it's marked
# by the comment "from swift".

from __future__ import with_statement

from ConfigParser import ConfigParser
import errno
import logging
import logging.handlers
from optparse import OptionParser
import pwd
import os
from random import random
import signal
import socket
import sys
import time

# from swift
class LoggerFileObject(object):
    """
    Used to capture stderr/stdout.
    """

    def __init__(self, logger):
        self.logger = logger

    def write(self, value):
        value = value.strip()
        if value:
            if 'Connection reset by peer' in value:
                self.logger.error('STDOUT: Connection reset by peer')
            else:
                self.logger.error('STDOUT: %s', value)

    def writelines(self, values):
        self.logger.error('STDOUT: %s', '#012'.join(values))

    def close(self):
        pass

    def flush(self):
        pass

    def __iter__(self):
        return self

    def next(self):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def read(self, size=-1):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def readline(self, size=-1):
        raise IOError(errno.EBADF, 'Bad file descriptor')

    def tell(self):
        return 0

    def xreadlines(self):
        return self

# from swift
def drop_privileges(user):
    """
    Sets the userid/groupid of the current process, get session leader, etc.

    :param user: User name to change privileges to
    """
    user = pwd.getpwnam(user)
    if os.geteuid() == 0:
        os.setgroups([])
    os.setgid(user[3])
    os.setuid(user[2])
    os.environ['HOME'] = user[5]
    try:
        os.setsid()
    except OSError:
        pass
    os.chdir('/')   # in case you need to rmdir on where you started the daemon
    os.umask(0o22)  # ensure files are created with the correct privileges

def get_command_line(command_line, dargs_parser, args_parser):
    """
    Parses the command line.

    Command line should be of the form:
    [common daemon args] command [unique daemon args].

    Returns common daemon args, command, and unique daemon args.
    """
    dargs = dargs_parser.parse_args(command_line)
    command = dargs[1][0]
    if command not in Daemon.commands:
        raise ValueError('Invalid daemon command')
    args = args_parser.parse_args(dargs[1][1:])
    return dargs, command, args

def get_project_from_conf_path(conf_path):
    if not os.path.isfile(conf_path):
        raise ValueError('File path expected')

    conf_file = os.path.basename(conf_path)

    if not conf_file.endswith('.conf'):
        raise ValueError('Conf file should end with .conf')

    return conf_file[:-len('.conf')]

# from swift
def list_from_csv(comma_separated_str):
    """
    Splits the str given and returns a properly stripped list of the comma
    separated values.
    """
    if comma_separated_str:
        return [v.strip() for v in comma_separated_str.split(',') if v.strip()]
    return []

def parse_run_name():
    """
    Returns the parts of the run name of the daemon.

    The run name should be of the form: project-daemon

    This is used to determine the config location/section name.
    """
    command = os.path.split(sys.argv[0])[1]
    parts = command.split('-')
    if len(parts) != 2:
        raise ValueError()
    return parts

# from swift
def read_config(conf_path):
    """
    Reads a config and returns its sections/values.
    """
    c = ConfigParser()
    if not c.read(conf_path):
        print "Unable to read config from %s" % conf_path
        sys.exit(1)
    conf = {}
    for s in c.sections():
        conf.update({s: dict(c.items(s))})
    conf['__file__'] = conf_path
    return conf

class Daemon(object):
    """
    A class for building daemons.

    It takes care of things common to all daemons.
    """

    commands = 'restart run_once start status stop'.split()
    handler4logger = {}

    def __init__(self, global_conf, conf_section, pid_file_path, dargs, args):
        self.global_conf = global_conf
        self.conf_section = conf_section
        self.conf = self.global_conf[conf_section]
        self.pid_file_path = pid_file_path
        self.dargs = dargs
        self.args = args
        self.logger = self.get_logger(self.conf)
        self.user = self.conf['user']
        self.interval = int(self.conf.get('interval', 5))

    # from swift
    def capture_stdio(self):
        """
        Log unhandled exceptions, close stdio, capture stdout and stderr.
        """
        # log uncaught exceptions
        sys.excepthook = lambda * exc_info: \
            self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)

        # collect stdio file desc not in use for logging
        stdio_files = [sys.stdin, sys.stdout, sys.stderr]

        with open(os.devnull, 'r+b') as nullfile:
            # close stdio (excludes fds open for logging)
            for f in stdio_files:
                # some platforms throw an error when attempting an stdin flush
                try:
                    f.flush()
                except IOError:
                    pass

                try:
                    os.dup2(nullfile.fileno(), f.fileno())
                except OSError:
                    pass

        # FUTURE: make the capture optional?
        sys.stdout = LoggerFileObject(self.logger)
        sys.stderr = LoggerFileObject(self.logger)

    def daemonize(self):
        """
        Daemonizes the current process.

        Returns False if the process is not the daemon.

        Returns True if process is the daemon.
        """
        try:
            pid = os.fork()
            if pid > 0:
                return False
        except Exception:
            raise RuntimeError('Fork failed')

        drop_privileges(self.user)

        def kill_children(*args):
            signal.signal(signal.SIGTERM, signal.SIG_IGN)
            os.killpg(0, signal.SIGTERM)
            sys.exit()

        signal.signal(signal.SIGTERM, kill_children)

        # FUTURE: something that logs when the daemon is killed?

        self.capture_stdio()

        with open(self.pid_file_path, 'w+') as fd:
            pid = os.getpid()
            fd.write('%d\n' % pid)

        return True

    @classmethod
    def get_args_parser(cls):
        """
        Override to parse options unique to your daemon.

        Returns an OptionParser.
        """
        return OptionParser()

    @classmethod
    def get_dargs_parser(cls):
        """
        Returns an OptionParse for options common to all daemons.

        Returns an OptionParser.
        """
        # FUTURE: add things that can be overridden on command line
        parser = OptionParser()
        parser.add_option(
            "--eventlet_patch", action="store_false", dest="eventlet_patch",
            default=False, help="add eventlet patch")
        parser.disable_interspersed_args()
        return parser

    # from swift
    @classmethod
    def get_logger(cls, conf):
        """
        Returns a logger configured from the conf.
        """
        if not conf:
            conf = {}
        name = conf.get('log_name', 'daemonx')
        log_route = conf.get('log_route', name)
        logger = logging.getLogger(log_route)
        logger.propagate = False

        # get_logger will only ever add one SysLog Handler to a logger
        if logger in cls.handler4logger:
            logger.removeHandler(cls.handler4logger[logger])

        # facility for this logger will be set by last call wins
        facility = getattr(
            logging.handlers.SysLogHandler,
            conf.get('log_facility', 'LOG_LOCAL0'),
            logging.handlers.SysLogHandler.LOG_LOCAL0)
        udp_host = conf.get('log_udp_host')
        if udp_host:
            udp_port = int(
                conf.get('log_udp_port',
                logging.handlers.SYSLOG_UDP_PORT))
            handler = logging.handlers.SysLogHandler(
                address=(udp_host, udp_port), facility=facility)
        else:
            log_address = conf.get('log_address', '/dev/log')
            try:
                handler = logging.handlers.SysLogHandler(
                    address=log_address, facility=facility)
            except socket.error, e:
                # Either /dev/log isn't a UNIX socket or it does not exist
                # at all
                if e.errno not in [errno.ENOTSOCK, errno.ENOENT]:
                    raise e
                handler = logging.handlers.SysLogHandler(facility=facility)
        logger.addHandler(handler)
        cls.handler4logger[logger] = handler

        # set the level for the logger
        logger.setLevel(
            getattr(
                logging, conf.get('log_level', 'INFO').upper(), logging.INFO))

        return logger

    def get_pid(self):
        """
        Reads and returns the daemon's pid from pid file on disk>

        Returns None on failure.
        """
        try:
            with open(self.pid_file_path, 'r') as fd:
                pid = int(fd.read().strip())
        except IOError:
            pid = None
        return pid

    def run(self):
        """
        Runs the daemon.

        It calls run_forever().
        """
        self.run_forever()

    @classmethod
    def run_command(
            cls, conf_path, conf_section, command_line, project=None,
            daemon_name=None):
        """
        Sends the command specified on the command line to the daemon.
        """
        # read config
        global_conf = read_config(conf_path)

        # get project/daemon name
        if not (project and daemon_name):
            project = get_project_from_conf_path(conf_path)
            daemon_name = conf_section

        # get/import class from config
        import_target, class_name = \
            global_conf[conf_section]['class'].rsplit('.', 1)
        module = __import__(import_target, fromlist=[import_target])
        daemon_class = getattr(module, class_name)

        # parse command line, get command to run on daemon
        dargs_parser = cls.get_dargs_parser()
        args_parser = cls.get_args_parser()
        dargs, command, args = get_command_line(
            command_line, dargs_parser, args_parser)

        # check command
        if command not in cls.commands:
            raise ValueError('Invalid command')

        # get pid file path
        pid_file_path = '/var/run/%s/%s.pid' % (project, daemon_name)

        # create daemon
        daemon = daemon_class(
            global_conf, conf_section, pid_file_path, dargs, args)

        # get and run method for command
        method = getattr(daemon, command)
        try:
            method()
        except Exception, e:
            print e

    @classmethod
    def run_command_from_script(cls):
        """
        Runs the command on the daemon.

        Project and daemon name are determined from the script run.
        """
        project, daemon_name = parse_run_name()
        conf_path = '/etc/%s/%s.conf' % (project, project)
        cls.run_command(
            conf_path, daemon_name, list(sys.argv[1:]), project, daemon_name)

    def run_forever(self):
        """
        Run the daemon forever.

        Sleeps as need be to not run more than once in each interval.

        Calls run_once().
        """
        time.sleep(random() * self.interval)
        while True:
            try:
                self.run_once()
            except Exception:
                self.logger.exception('run_once()')
            time.sleep(self.interval)

    def restart(self):
        """
        Restarts the daemon.
        """
        self.stop()
        self.start()

    def run_once(self):
        """
        Override this to define what the daemon does.
        """
        raise NotImplementedError('run_once not implemented')

    def start(self):
        """
        Starts the daemon.
        """
        pid = self.get_pid()
        if pid:
            raise RuntimeError('Daemon alredy running')

        if self.daemonize():
            self.run()

    def status(self):
        """
        Prints the status of the daemon.
        """
        pid = self.get_pid()
        if pid:
            print 'Daemon is running with pid: %d' % pid
        else:
            print 'Daemon is not running'

    def stop(self):
        """
        Stops the daemon.
        """
        pid = self.get_pid()

        if not pid:
            print 'Daemon does not seem to be running'
            return

        try:
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(.1)
        except OSError, e:
            if str(e).find('No such process') > 0:
                if os.path.exists(self.pid_file_path):
                    os.remove(self.pid_file_path)
                else:
                    raise
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Convert data read by something like csv.DictReader to GeoJSON on the fly
"""

import csv
import logging
import json

import click
import fiona
import shapely.geometry
import shapely.wkt

log = logging.getLogger('csv_adapter')

def helper_str(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value
    is cast to a string and returned.
    """

    if val is None or val == '':
        return None
    else:
        return str(val)

def helper_float(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value is
    cast to a float and returned.
    """

    if val is None or val == '':
        return None
    else:
        return float(val)

def helper_int(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value is
    cast to an int and returned.
    """

    if val is None or val == '':
        return None

def helper_properties_def(properties):

    """
    Convert properties from a Fiona datasource to a definition that is usable by
    `dict_reader_as_geojson` and utilizes the helper functions for better value
    handling.

    Sample input:

        {
            'float_field': 'float:14',
            'int_field': 'int:8',
            'str_field': 'str:16'
        }

    Sample output:

        {
            'float_field': helper_float,
            'int_field': helper_int,
            'str_field': helper_str
        }

    Parameters
    ----------
    properties : dict
        Dictionary from a Fiona datasource.

    Returns
    -------
    dict
        Keys are fields from the input properties and vals are one of the three
        helper functions: helper_str, helper_int, helper_float.
    """

    return {field: globals()['helper_' + properties[field].split(':')[0]] for field in properties}

def dict_reader_as_geojson(dict_reader, geomtype_field, properties=None, skip_failures=False, empty_is_none=True):

    """
    Generator to read a CSV containing vector data and convert to GeoJSON on
    the fly.  Geometry can be supplied as WKT, GeoJSON, or points with X and
    Y stored in different fields.

    The user supplies an iterable, like an instance of `csv.DictReader()`, that
    provides one dictionary per iteration that is converted on the fly to a
    GeoJSON object and yielded to the parent process.  The user can specify
    which field contains the geometry and what kind of geometry representation
    it contains.

    All keys except `None` or the geometry field are included in the output
    features but the user can specify what fields to include and what type
    they should be cast to via the `properties` parameter.  If set to `None`
    all fields are included in the output feature but if set to a dictionary
    where keys are fieldnames and values are functions then only the fields
    specified in the keys are included in the output.  The functions must
    take a single argument, return a single value, and be prepared to handle
    values contained in the associated field in the dictionaries yielded by
    the `dict_reader`.

    Helper functions have been included to make creating a `properties`
    definition containing `str`, `float`, or `int` easier.  They are prefixed
    with `helper_` and `helper_properties_def` takes a set of Fiona properties
    and returns a definition suitable for the `properties` parameter that uses
    the helper functions.

    Example `properties` definition:

        {
            'field1': float,
            'field2': helper_int,
            'field3': custom_function
        }

    Create a `properties` definition from a set of Fiona properties:

    Valid `geomtype_field` definitions:

        wkt:field
        geojson:field
        xy:x_field,y_field[,z_field]

    Parameters
    ----------
    dict_reader : csv.DictReader (or other iterable object returning dicts)
        A loaded instance configured with whatever settings the user desires.
    skip_failures : bool, optional
        If `True` and an exception is encountered while iterating over rows, that
        row will be skipped and the exception will be logged.  If `False` the
        exception is raised.
    geomtype_field : str
        Geometry type and field description.
    properties : dict, optional
        Keys are fieldnames in the CSV and vals are functions used to cast the
        values to a Python type.  If `None` then all fields are converted to
        strings.  Fields that are `None` or are specified in the
        `geomtype_field` argument are ignored even if they are specified in the
        `properties` parameter.

    Yields
    ------
    dict
        A GeoJSON feature.
    """

    _allowed_geomtypes = ('wkt', 'geojson', 'xy')

    # Parse the geometry field definition
    geom_type, geom_field = geomtype_field.split(':')
    geom_type = geom_type.lower()

    # Fail fast in case streaming from a large file
    if geom_type not in _allowed_geomtypes:
        raise ValueError("Invalid geometry type - must be one of: {gt}".format(gt=_allowed_geomtypes))
    elif geom_type == 'xy' and ',' not in geom_field:
        raise ValueError("Geometry type is 'xy' and ',' does not appear in fieldnames: %s" % geom_field)

    for row_num, _row in enumerate(dict_reader):

        # Define the properties definition from the first row if the user didn't supply one
        if properties is None:
            properties = {field: helper_str for field in _row if field not in (None, geom_field)}

        # skip_failures option only skips failures that occur within this block
        try:

            # Convert vals that are empty strings to None
            row = {}
            if empty_is_none:
                for k, v in _row.items():
                    if v == '':
                        v = None
                    row[k] = v
            else:
                row = _row

            # Convert string representation of geometry to a geojson object
            if geom_type == 'wkt':
                geometry = shapely.geometry.mapping(shapely.wkt.loads(row[geom_field]))
            elif geom_type == 'geojson':
                geometry = json.loads(row[geom_field])
            elif geom_type == 'xy':
                geometry = {
                    "type": "Point",
                    "coordinates": [float(row[field]) for field in geom_field.split(',')]
                }
            else:
                raise ValueError("Invalid geometry type definition: %s" % geom_type)

            # Give the parent process a GeoJSON feature
            yield {
                "id": row_num,
                "type": "Feature",
                "geometry": geometry,
                "properties": {field: caster(row[field]) for field, caster in properties.items()}
            }

        # Encountered an error
        except Exception as e:
            if skip_failures:
                log.exception(e)
            else:
                raise e

def _newlinejson_reader(infile):

    """
    Read a newline delimited JSON file

    Parameters
    ----------
    infile : file
        Open file-like object with read access.

    Yields
    ------
    dict or list
    """

    for line in infile:
        if line:
            yield json.loads(line)

@click.command()
@click.argument('infile', type=click.File(mode='r'), required=True)
@click.argument('outfile', default='-', required=True)
@click.option(
    '-sf', '--skip-failures', type=click.BOOL,
    help="Skip failures encountered on read and write."
)
@click.option(
    '-r', '--reader', default='csv', type=click.Choice(['csv', 'json', 'newlinejson']),
    help="Input data format."
)
@click.option(
    '-f', '--format', '--driver', required=True,
    help="Output driver name."
)
@click.option(
    '-gf', '--geom-field', required=True,
    help="Geometry field definition as `format:field'."
)
@click.option(
    '-co', '--creation-option', multiple=True,
    help="Driver specific creation option as key=val."
)
@click.option(
    '-p', '--property-def', multiple=True,
    help="Schema property as field=def."
)
@click.option(
    '-crs', '--crs',
    help="Specify CRS of input data."
)
def main(infile, outfile, creation_option, skip_failures, reader, driver, geom_def, property_def, crs):

    """
    Convert delimited vector data to an OGR datasource.
    """

    def first_plus_iterator(first, iterator):
        if first:
            _first = first
            first = None
            yield _first
        else:
            for item in iterator:
                yield item

    creation_option = {co.split('=')[0]: co.split('=')[1] for co in creation_option}
    properties = {fd.split('=')[0]: fd.split('=')[1] for fd in property_def}

    # Load the input file into a generator that produces one GeoJSON feature per iteration
    if reader == 'csv':
        reader = csv.DictReader(infile),
    elif reader == 'json':
        reader = json.load(infile)
    elif reader == 'newlinejson':
        reader = _newlinejson_reader(infile)
    feature_generator = dict_reader_as_geojson(reader, geom_def, properties=helper_properties_def(properties),
                                               skip_failures=skip_failures)

    # Cache the first feature so that
    first_feature = next(feature_generator)
    schema = first_feature['geometry']['type']
    if crs.startswith('+'):
        schema['']
    with fiona.open(outfile, 'w', driver=driver, schema=None):
        pass

if __name__ == '__main__':
    main()
<commit_msg>Functional<commit_after>#!/usr/bin/env python

"""
Convert data read by something like csv.DictReader to GeoJSON on the fly.

Names are overly verbose for ease of following what does what.
"""

import csv
import logging
import json
import sys

import click
import fiona
import fiona.transform
import shapely.geometry
import shapely.wkt

# Some OGR fields can contain valid data the exceeds the default CSV field size
csv.field_size_limit(sys.maxsize)

def helper_str(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value
    is cast to a string and returned.
    """

    if val is None or val == '':
        return None
    else:
        return str(val)

def helper_float(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value is
    cast to a float and returned.
    """

    if val is None or val == '':
        return None
    else:
        return float(val)

def helper_int(val):

    """
    Helper function for use with `dict_reader_as_geojson`.  Returns `None` if
    the input value is an empty string or `None`, otherwise the input value is
    cast to an int and returned.
    """

    if val is None or val == '':
        return None
    else:
        return int(val)

def helper_properties_def(properties):

    """
    Convert properties from a Fiona datasource to a definition that is usable by
    `dict_reader_as_geojson` and utilizes the helper functions for better value
    handling.

    Sample input:

        {
            'float_field': 'float:14',
            'int_field': 'int:8',
            'str_field': 'str:16'
        }

    Sample output:

        {
            'float_field': helper_float,
            'int_field': helper_int,
            'str_field': helper_str
        }

    Parameters
    ----------
    properties : dict
        Dictionary from a Fiona datasource.

    Returns
    -------
    dict
        Keys are fields from the input properties and vals are one of the three
        helper functions: helper_str, helper_int, helper_float.
    """

    return {field: globals()['helper_' + properties[field].split(':')[0]] for field in properties}

def dict_reader_as_geojson(dict_reader, geomtype_field, properties=None, skip_failures=False, empty_is_none=True):

    """
    Generator to read a CSV containing vector data and convert to GeoJSON on
    the fly.  Geometry can be supplied as WKT, GeoJSON, or points with X and
    Y stored in different fields.

    The user supplies an iterable, like an instance of `csv.DictReader()`, that
    provides one dictionary per iteration that is converted on the fly to a
    GeoJSON object and yielded to the parent process.  The user can specify
    which field contains the geometry and what kind of geometry representation
    it contains.

    All keys except `None` or the geometry field are included in the output
    features but the user can specify what fields to include and what type
    they should be cast to via the `properties` parameter.  If set to `None`
    all fields are included in the output feature but if set to a dictionary
    where keys are fieldnames and values are functions then only the fields
    specified in the keys are included in the output.  The functions must
    take a single argument, return a single value, and be prepared to handle
    values contained in the associated field in the dictionaries yielded by
    the `dict_reader`.

    Helper functions have been included to make creating a `properties`
    definition containing `str`, `float`, or `int` easier.  They are prefixed
    with `helper_` and `helper_properties_def` takes a set of Fiona properties
    and returns a definition suitable for the `properties` parameter that uses
    the helper functions.

    Example `properties` definition:

        {
            'field1': float,
            'field2': helper_int,
            'field3': custom_function
        }

    Create a `properties` definition from a set of Fiona properties:

    Valid `geomtype_field` definitions:

        wkt:field
        geojson:field
        xy:x_field,y_field[,z_field]

    Parameters
    ----------
    dict_reader : csv.DictReader (or other iterable object returning dicts)
        A loaded instance configured with whatever settings the user desires.
    skip_failures : bool, optional
        If `True` and an exception is encountered while iterating over rows, that
        row will be skipped and the exception will be logged.  If `False` the
        exception is raised.
    geomtype_field : str
        Geometry type and field description.
    properties : dict, optional
        Keys are fieldnames in the CSV and vals are functions used to cast the
        values to a Python type.  If `None` then all fields are converted to
        strings.  Fields that are `None` or are specified in the
        `geomtype_field` argument are ignored even if they are specified in the
        `properties` parameter.

    Yields
    ------
    dict
        A GeoJSON feature.
    """

    _allowed_geomtypes = ('wkt', 'geojson', 'xy')

    # Parse the geometry field definition
    geom_type, geometry_field = geomtype_field.split(':')
    geom_type = geom_type.lower()

    # Fail fast in case streaming from a large file
    if geom_type not in _allowed_geomtypes:
        raise ValueError("Invalid geometry type - must be one of: {gt}".format(gt=_allowed_geomtypes))
    elif geom_type == 'xy' and ',' not in geometry_field:
        raise ValueError("Geometry type is 'xy' and ',' does not appear in fieldnames: %s" % geometry_field)

    for row_num, _row in enumerate(dict_reader):

        # Define the properties definition from the first row if the user didn't supply one
        if properties is None:
            properties = {field: helper_str for field in _row if field not in (None, geometry_field)}

        # skip_failures option only skips failures that occur within this block
        try:

            # Convert vals that are empty strings to None
            row = {}
            if empty_is_none:
                for k, v in _row.items():
                    if v == '':
                        v = None
                    row[k] = v
            else:
                row = _row

            # Convert string representation of geometry to a geojson object
            if geom_type == 'wkt':
                geometry = shapely.geometry.mapping(shapely.wkt.loads(row[geometry_field]))
            elif geom_type == 'geojson':
                geometry = json.loads(row[geometry_field])
            elif geom_type == 'xy':
                geometry = {
                    "type": "Point",
                    "coordinates": [float(row[field]) for field in geometry_field.split(',')]
                }
            else:
                raise ValueError("Invalid geometry type definition: %s" % geom_type)

            # Give the parent process a GeoJSON feature
            yield {
                "id": row_num,
                "type": "Feature",
                "geometry": geometry,
                "properties": {field: caster(row[field]) for field, caster in properties.items()}
            }

        # Encountered an error
        except Exception as e:
            if skip_failures:
                logging.exception(str(e))
            else:
                raise e

def _newlinejson_reader(infile):

    """
    Read a newline delimited JSON file

    Parameters
    ----------
    infile : file
        Open file-like object with read access.

    Yields
    ------
    dict or list
    """

    for line in infile:
        if line:
            yield json.loads(line)

@click.command()
@click.argument('infile', type=click.File(mode='r'), required=True)
@click.argument('outfile', required=True)
@click.option(
    '-sf', '--skip-failures', is_flag=True, type=click.BOOL,
    help="Skip failures encountered on read and write."
)
@click.option(
    '-r', '--reader', metavar='NAME', default='csv', type=click.Choice(['csv', 'json', 'newlinejson']),
    help="Input data format."
)
@click.option(
    '-f', '--format', '--driver', metavar='NAME',
    help="Output driver name."
)
@click.option(
    '-gf', '--geometry-field', metavar='NAME', required=True,
    help="Geometry field definition as `format:field'."
)
@click.option(
    '-co', '--creation-option', metavar='NAME=VAL', multiple=True,
    help="Driver specific creation option as key=val."
)
@click.option(
    '-p', '--property-definition', metavar='FIELD=TYPE:WIDTH', multiple=True,
    help="Schema property as field=def."
)
@click.option(
    '-s-crs', '--src_crs', metavar='CRS_DEF',
    help="Specify CRS of input data."
)
@click.option(
    '-d-crs', '--dst_crs', metavar='CRS_DEF',
    help="Specify CRS of output data."
)
@click.option(
    '-sl', '--skip-lines', metavar='N', type=click.INT, default=0,
    help="Skip N lines."
)
@click.option(
    '-ss', '--subsample', metavar='N', type=click.INT, default=0,
    help="Only process N lines."
)
@click.option(
    '-gt', '--geometry-type', metavar='TYPE',
    help="Specify geometry type for output layer."
)
def main(infile, outfile, creation_option, skip_failures, reader, driver, geometry_field, property_definition,
         src_crs, dst_crs, skip_lines, subsample, geometry_type):

    """
    Convert delimited vector data to an OGR datasource.

    Print GeoJSON to stdout:

        $ ./delimited2datasource.py sample-data/WV.csv - -gf wkt:WKT

    Convert to a shapefile:

        $ ./delimited2datasource.py sample-data/WV.csv WV.shp -gf wkt:WKT

    Skip 5 lines and only process the next 10:

        $ ./delimited2datasource.py sample-data/WV.csv - -gf wkt:WKT -sl 5 -ss 10

    Reproject geometries:

        $ ./delimited2datasource.py sample-data/WV.csv - -gf wkt:WKT \
            --src_crs EPSG:4269 --dst_crs EPSG:4326

    Only write two fields but convert one to an int:

        $ ./delimited2datasource.py sample-data/WV.csv - \
            -gf wkt:WKT -p NAME=str -p STATEFP=int

    Read point data from an x and y columns while casting one field to int and one to float:

        $ ./delimited2datasource.py sample-data/WV.csv - \
            -gf xy:centroid_x,centroid_y -p COUNTYFP=int -p ALAND=float
    """

    def first_plus_iterator(first, iterator):
        yielded_first = False
        for item in iterator:
            if not yielded_first:
                yielded_first = True
                yield first
            else:
                yield item

    if outfile == '-' and driver is not None or len(creation_option) > 0:
        click.echo("ERROR: Cannot specify driver or creation options if printing to stdout.", err=True)
        sys.exit(1)

    # Convert the input file into an iterable object
    if reader == 'csv':
        reader = csv.DictReader(infile)
    elif reader == 'json':
        reader = json.load(infile)
    elif reader == 'newlinejson':
        reader = _newlinejson_reader(infile)
    else:
        raise ValueError("Invalid reader: `%s'" % reader)

    # If the user doesn't supply any properties then the transformer will automatically generate them from the first
    # feature.  The first output GeoJSON feature can be cached and a set of Fiona properties can be constructed from it.
    # Kinda sketchy and could use a re-write
    properties = {p.split('=')[0]: p.split('=')[1] for p in property_definition}
    if not properties:
        generator_properties = None
    else:
        generator_properties = helper_properties_def(properties)
    feature_generator = dict_reader_as_geojson(reader, geometry_field, properties=generator_properties,
                                               skip_failures=skip_failures)

    # Cache the first feature so we can extract information to build the schema
    first_feature = next(feature_generator)

    # Build the schema for the output Fiona datasource
    schema = {co.split('=')[0]: co.split('=')[1] for co in creation_option}
    if geometry_type is None:
        geometry_type = first_feature['geometry']['type']
    if properties is None:
        properties = {p: 'str' for p in first_feature['properties'].keys()}
    schema.update(
        crs=dst_crs,
        driver=driver,
        schema={
            'geometry': geometry_type,
            'properties': properties
        }
    )
    with fiona.open(outfile, 'w', **schema) if outfile != '-' else sys.stdout as dst:

        for idx, feature in enumerate(first_plus_iterator(first_feature, feature_generator)):

            if idx >= skip_lines:

                # Skip failures block
                try:

                    feature['geometry'] = fiona.transform.transform_geom(
                        src_crs, dst_crs, feature['geometry'], antimeridian_cutting=True)

                    if dst is sys.stdout:
                        click.echo(json.dumps(feature))
                    else:
                        dst.write(feature)

                    # Subsample and skip lines are 1 indexing while idx is 0 indexing so add 1 for the comparison since
                    # its always off
                    if idx + 1 is subsample + skip_lines:
                        break

                except Exception as e:
                    if not skip_failures:
                        raise e

    sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>