<commit_before>#!/usr/bin/env python

# Based on previous work by
# Charles Menguy (see: http://stackoverflow.com/questions/10217067/implementing-a-full-python-unix-style-daemon-process)
# and Sander Marechal (see: http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/)

# Adapted by M.Hendrix [2015,2016]

# daemon97.py pushes data to the MySQL-server.
# daemon23 support

import syslog, traceback
import os, sys, shutil, glob, time, subprocess
from libdaemon import Daemon
import ConfigParser
import MySQLdb as mdb

DEBUG = False
IS_SYSTEMD = os.path.isfile('/bin/journalctl')
leaf = os.path.realpath(__file__).split('/')[-2]

class MyDaemon(Daemon):
  def run(self):
    try:              # Initialise MySQLdb
      consql = mdb.connect(host='sql.lan', db='domotica', read_default_file='~/.my.cnf')

      if consql.open: # Hardware initialised succesfully -> get a cursor on the DB.
        cursql = consql.cursor()
        cursql.execute("SELECT VERSION()")
        versql = cursql.fetchone()
        cursql.close()
        logtext = "{0} : {1}".format("Attached to MySQL server", versql)
        syslog.syslog(syslog.LOG_INFO, logtext)
    except mdb.Error, e:
      if DEBUG:
        print "Unexpected MySQL error"
        print "Error {0:d}: {1!s}".format(e.args[0], e.args[1])
      if consql:    # attempt to close connection to MySQLdb
        if DEBUG:print "Closing MySQL connection"
        consql.close()
        syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
      syslog.syslog(syslog.LOG_ALERT,e.__doc__)
      syslog_trace(traceback.format_exc())
      raise

    iniconf = ConfigParser.ConfigParser()
    inisection = "97"
    home = os.path.expanduser('~')
    s = iniconf.read(home + '/' + leaf + '/config.ini')
    if DEBUG: print "config file : ", s
    if DEBUG: print iniconf.items(inisection)
    reportTime = iniconf.getint(inisection, "reporttime")
    cycles = iniconf.getint(inisection, "cycles")
    samplesperCycle = iniconf.getint(inisection, "samplespercycle")
    flock = iniconf.get(inisection, "lockfile")

    samples = samplesperCycle * cycles              # total number of samples averaged
    sampleTime = reportTime/samplesperCycle         # time [s] between samples
    cycleTime = samples * sampleTime                # time [s] per cycle

    myname = os.uname()[1]
    while True:
      try:
        startTime=time.time()

        do_sql_data(flock, iniconf, consql)

        waitTime = sampleTime - (time.time() - startTime) - (startTime%sampleTime)
        if (waitTime > 0):
          if DEBUG:print "Waiting {0} s".format(waitTime)
          time.sleep(waitTime)

      except Exception as e:
        if DEBUG:
          print "Unexpected error:"
          print e.message
        # attempt to close connection to MySQLdb
        if consql:
          if DEBUG:print "Closing MySQL connection"
          consql.close()
          syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
        syslog.syslog(syslog.LOG_ALERT,e.__doc__)
        syslog_trace(traceback.format_exc())
        raise

def cat(filename):
  ret = ""
  if os.path.isfile(filename):
    with open(filename,'r') as f:
      ret = f.read().strip('\n')
  return ret

def do_writesample(cnsql, cmd, sample):
  fail2write = False
  dat = (sample.split(', '))
  try:
    cursql = cnsql.cursor()
    if DEBUG:print "   ",dat
    cursql.execute(cmd, dat)
    cnsql.commit()
    cursql.close()
  #except OperationalError as e:
  #  fail2write=False
  #  reconnect()
  #  if DEBUG: print e
  #  syslog.syslog(syslog.LOG_ALERT,"********* Raising!!")
  #  syslog.syslog(syslog.LOG_ALERT,e.__doc__)
  #  syslog_trace(traceback.format_exc())
  #  raise
  except Exception as e:
    fail2write=False
    if DEBUG:
      print "Unexpected error in do_writesample:"
      print e.message
    # attempt to close connection to MySQLdb
    if consql:
      if DEBUG:print "Closing MySQL connection"
      consql.close()
      syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
    syslog.syslog(syslog.LOG_ALERT,e.__doc__)
    syslog_trace(traceback.format_exc())
    raise

  return fail2write

def do_sql_data(flock, inicnfg, cnsql):
  if DEBUG:print "Pushing data to MySQL-server"
  # set a lock
  lock(flock)
  time.sleep(2)
  # wait for all other processes to release their locks.
  count_internal_locks=2
  while (count_internal_locks > 1):
    time.sleep(1)
    count_internal_locks=0
    for fname in glob.glob(r'/tmp/' + leaf + '/*.lock'):
      count_internal_locks += 1
    if DEBUG:print "{0} internal locks exist".format(count_internal_locks)
  #endwhile

  for inisect in inicnfg.sections(): # Check each section of the config.ini file
    errsql = False
    try:
      ifile = inicnfg.get(inisect,"resultfile")
      if DEBUG:print ifile

      try:
        sqlcmd = []
        sqlcmd = inicnfg.get(inisect,"sqlcmd")
        if DEBUG:print sqlcmd

        data = cat(ifile).splitlines()
        if data:
          for entry in range(0, len(data)):
            errsql = do_writesample(cnsql, sqlcmd, data[entry])
          #endfor
        #endif
      except Exception as e:  #no sqlcmd
        if DEBUG:
          print "Unexpected error:"
          print e.message
    except Exception as e:  #no ifile
      if DEBUG:
        print "Unexpected error:"
        print e.message

    try:
      ofile = inicnfg.get(inisect,"rawfile")
      if DEBUG:print ofile
      if not errsql:                    # SQL-job was successful or non-existing
        if os.path.isfile(ifile):       # IF resultfile exists
          if not os.path.isfile(ofile): # AND rawfile does not exist
            shutil.move(ifile, ofile)   # THEN move the file over
    except Exception as e:  #no ofile
      if DEBUG:
        print "Unexpected error:"
        print e.message

  #endfor
  unlock(flock)

def lock(fname):
  open(fname, 'a').close()

def unlock(fname):
  if os.path.isfile(fname):
    os.remove(fname)

def syslog_trace(trace):
  #Log a python stack trace to syslog
  log_lines = trace.split('\n')
  for line in log_lines:
    if line:
      syslog.syslog(syslog.LOG_ALERT,line)

if __name__ == "__main__":
  daemon = MyDaemon('/tmp/' + leaf + '/97.pid')
  if len(sys.argv) == 2:
    if 'start' == sys.argv[1]:
      daemon.start()
    elif 'stop' == sys.argv[1]:
      daemon.stop()
    elif 'restart' == sys.argv[1]:
      daemon.restart()
    elif 'foreground' == sys.argv[1]:
      # assist with debugging.
      print "Debug-mode started. Use <Ctrl>+C to stop."
      DEBUG = True
      if DEBUG:
        logtext = "Daemon logging is ON"
        syslog.syslog(syslog.LOG_DEBUG, logtext)
      daemon.run()
    else:
      print "Unknown command"
      sys.exit(2)
    sys.exit(0)
  else:
    print "usage: {0!s} start|stop|restart|foreground".format(sys.argv[0])
    sys.exit(2)
<commit_msg>20160226T1539<commit_after>#!/usr/bin/env python

# Based on previous work by
# Charles Menguy (see: http://stackoverflow.com/questions/10217067/implementing-a-full-python-unix-style-daemon-process)
# and Sander Marechal (see: http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/)

# Adapted by M.Hendrix [2015,2016]

# daemon97.py pushes data to the MySQL-server.
# daemon23 support

import syslog, traceback
import os, sys, shutil, glob, time, subprocess
from libdaemon import Daemon
import ConfigParser
import MySQLdb as mdb

DEBUG = False
IS_SYSTEMD = os.path.isfile('/bin/journalctl')
leaf = os.path.realpath(__file__).split('/')[-2]

class MyDaemon(Daemon):
  def run(self):
    try:              # Initialise MySQLdb
      consql = mdb.connect(host='sql.lan', db='domotica', read_default_file='~/.my.cnf')

      if consql.open: # Hardware initialised succesfully -> get a cursor on the DB.
        cursql = consql.cursor()
        cursql.execute("SELECT VERSION()")
        versql = cursql.fetchone()
        cursql.close()
        logtext = "{0} : {1}".format("Attached to MySQL server", versql)
        syslog.syslog(syslog.LOG_INFO, logtext)
    except mdb.Error, e:
      if DEBUG:
        print "Unexpected MySQL error"
        print "Error {0:d}: {1!s}".format(e.args[0], e.args[1])
      if consql:    # attempt to close connection to MySQLdb
        if DEBUG:print "Closing MySQL connection"
        consql.close()
        syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
      syslog.syslog(syslog.LOG_ALERT,e.__doc__)
      syslog_trace(traceback.format_exc())
      raise

    iniconf = ConfigParser.ConfigParser()
    inisection = "97"
    home = os.path.expanduser('~')
    s = iniconf.read(home + '/' + leaf + '/config.ini')
    if DEBUG: print "config file : ", s
    if DEBUG: print iniconf.items(inisection)
    reportTime = iniconf.getint(inisection, "reporttime")
    cycles = iniconf.getint(inisection, "cycles")
    samplesperCycle = iniconf.getint(inisection, "samplespercycle")
    flock = iniconf.get(inisection, "lockfile")

    samples = samplesperCycle * cycles              # total number of samples averaged
    sampleTime = reportTime/samplesperCycle         # time [s] between samples
    cycleTime = samples * sampleTime                # time [s] per cycle

    myname = os.uname()[1]
    while True:
      try:
        startTime=time.time()

        do_sql_data(flock, iniconf, consql)

        waitTime = sampleTime - (time.time() - startTime) - (startTime%sampleTime)
        if (waitTime > 0):
          if DEBUG:print "Waiting {0} s".format(waitTime)
          time.sleep(waitTime)

      except Exception as e:
        if DEBUG:
          print "Unexpected error:"
          print e.message
        # attempt to close connection to MySQLdb
        if consql:
          if DEBUG:print "Closing MySQL connection"
          consql.close()
          syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
        syslog.syslog(syslog.LOG_ALERT,e.__doc__)
        syslog_trace(traceback.format_exc())
        raise

def cat(filename):
  ret = ""
  if os.path.isfile(filename):
    with open(filename,'r') as f:
      ret = f.read().strip('\n')
  return ret

def do_writesample(cnsql, cmd, sample):
  fail2write = False
  dat = (sample.split(', '))
  try:
    cursql = cnsql.cursor()
    if DEBUG:print "   ",dat
    cursql.execute(cmd, dat)
    cnsql.commit()
    cursql.close()
  #except OperationalError as e:
  #  fail2write=False
  #  reconnect()
  #  if DEBUG: print e
  #  syslog.syslog(syslog.LOG_ALERT,"********* Raising!!")
  #  syslog.syslog(syslog.LOG_ALERT,e.__doc__)
  #  syslog_trace(traceback.format_exc())
  #  raise
  except Exception as e:
    fail2write=False
    if DEBUG:
      print "Unexpected error in do_writesample:"
      print e.message
    # attempt to close connection to MySQLdb
    if cnsql:
      if DEBUG:print "Closing MySQL connection"
      cnsql.close()
      syslog.syslog(syslog.LOG_ALERT,"Closed MySQL connection")
    syslog.syslog(syslog.LOG_ALERT,e.__doc__)
    syslog_trace(traceback.format_exc())
    raise

  return fail2write

def do_sql_data(flock, inicnfg, cnsql):
  if DEBUG:print "Pushing data to MySQL-server"
  # set a lock
  lock(flock)
  time.sleep(2)
  # wait for all other processes to release their locks.
  count_internal_locks=2
  while (count_internal_locks > 1):
    time.sleep(1)
    count_internal_locks=0
    for fname in glob.glob(r'/tmp/' + leaf + '/*.lock'):
      count_internal_locks += 1
    if DEBUG:print "{0} internal locks exist".format(count_internal_locks)
  #endwhile

  for inisect in inicnfg.sections(): # Check each section of the config.ini file
    errsql = False
    try:
      ifile = inicnfg.get(inisect,"resultfile")
      if DEBUG:print ifile

      try:
        sqlcmd = []
        sqlcmd = inicnfg.get(inisect,"sqlcmd")
        if DEBUG:print sqlcmd

        data = cat(ifile).splitlines()
        if data:
          for entry in range(0, len(data)):
            errsql = do_writesample(cnsql, sqlcmd, data[entry])
          #endfor
        #endif
      except Exception as e:  #no sqlcmd
        if DEBUG:
          print "Unexpected error:"
          print e.message
    except Exception as e:  #no ifile
      if DEBUG:
        print "Unexpected error:"
        print e.message

    try:
      ofile = inicnfg.get(inisect,"rawfile")
      if DEBUG:print ofile
      if not errsql:                    # SQL-job was successful or non-existing
        if os.path.isfile(ifile):       # IF resultfile exists
          if not os.path.isfile(ofile): # AND rawfile does not exist
            shutil.move(ifile, ofile)   # THEN move the file over
    except Exception as e:  #no ofile
      if DEBUG:
        print "Unexpected error:"
        print e.message

  #endfor
  unlock(flock)

def lock(fname):
  open(fname, 'a').close()

def unlock(fname):
  if os.path.isfile(fname):
    os.remove(fname)

def syslog_trace(trace):
  #Log a python stack trace to syslog
  log_lines = trace.split('\n')
  for line in log_lines:
    if line:
      syslog.syslog(syslog.LOG_ALERT,line)

if __name__ == "__main__":
  daemon = MyDaemon('/tmp/' + leaf + '/97.pid')
  if len(sys.argv) == 2:
    if 'start' == sys.argv[1]:
      daemon.start()
    elif 'stop' == sys.argv[1]:
      daemon.stop()
    elif 'restart' == sys.argv[1]:
      daemon.restart()
    elif 'foreground' == sys.argv[1]:
      # assist with debugging.
      print "Debug-mode started. Use <Ctrl>+C to stop."
      DEBUG = True
      if DEBUG:
        logtext = "Daemon logging is ON"
        syslog.syslog(syslog.LOG_DEBUG, logtext)
      daemon.run()
    else:
      print "Unknown command"
      sys.exit(2)
    sys.exit(0)
  else:
    print "usage: {0!s} start|stop|restart|foreground".format(sys.argv[0])
    sys.exit(2)
<|endoftext|>


<commit_before>"""
Module creates draft JSON file(s) from given csv data sources that after user
review will be inserted into the meta.json (soon to be called table_info.json)

You can think of meta.json as the skeleton template for the data in tables
loaded into the database.
"""

import logging
import json
import sys
import os

import pandas as pandas

python_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                               os.pardir, os.pardir))
sys.path.append(python_filepath)

from housinginsights.ingestion.DataReader import ManifestReader

#configuration
#See /logs/example-logging.py for usage examples
logging_path = os.path.abspath(os.path.join(python_filepath, "logs"))
logging_filename = os.path.abspath(os.path.join(logging_path, "ingestion.log"))
logging.basicConfig(filename=logging_filename, level=logging.DEBUG)
#Pushes everything from the logger to the command line output as well
logging.getLogger().addHandler(logging.StreamHandler())

def _sql_name_clean(name):
    """
    Internal function to clean column names for the sql_name JSON field.

    :param name: column name for sql_nam JSON field
    :type name: str

    :return: cleaned up column name as lower case
    """
    # TODO: add code to make sure doesn't end or begin with underscore
    for item in ["-"," ","."]:
        if item in name:
            name = name.replace(item, "_")
    return name.lower()

def _pandas_to_sql_data_type(pandas_type_string):
    """
    An internal mapping function that returns the sql equivalent data type of a
    given pandas data type.

    :param pandas_type_string: pandas data type to look for
    :type pandas_type_string: str

    :return: sql equivalent data type
    """
    mapping = {
        'object': 'text',
        'int64': 'integer',
        'float64': 'decimal',
        'datetime64': 'timestamp'
    }
    # TODO: refactor to return key with default look up?
    try:
        sql_type = mapping[pandas_type_string]
    except KeyError:
        sql_type = 'text'
    return sql_type

def make_draft_json(filename, tablename, encoding):
    """
    Create a draft JSON file for the new table that will be reviewed and
    cleaned up before inserted into meta.json (soon to be called
    table_info.json).

    The draft output will be saved within '/housing-insights/python/logs'
    folder.

    :param filename: the filename of the csv data source
    :type filename: str

    :param tablename: the table name to be used in sql database
    :type tablename: str

    :param encoding: character encoding to pass to pandas - use 'latin1' or
    'utf-8' if that doesn't.
    :type encoding: str

    :return: None
    """

    # basic structure of the draft JSON output
    output = {
        tablename: {
            # TODO: change cleaner naming scheme to CamelCase
            "cleaner": tablename + "{}".format("_cleaner"),
            "replace_table": True,
            "fields": []
        }
    }

    # TODO: insert 'unique_data_id' field into draft JSON
    unique_data_id_field = {
        "display_name": "Unique data ID",
        "display_text": "Identifies which source file this record came from",
        "source_name": "unique_data_id",
        "sql_name": "unique_data_id",
        "type": "text",
        "required_in_source": False
    }
    output[tablename]["fields"].append(unique_data_id_field)

    # read csv file as pandas data frame and column headers
    dataframe_file = pandas.read_csv(filename, encoding=encoding)
    dataframe_iterator = dataframe_file.columns

    # Iterate through each field and populate draft JSON with attributes info.
    for field in dataframe_iterator:
        pandas_type = str(dataframe_file[field].dtypes)
        sql_type = _pandas_to_sql_data_type(pandas_type)

        field_attributes = {
                "type": sql_type,
                "source_name": field,
                "sql_name": _sql_name_clean(field),
                "display_name": _sql_name_clean(field),
                "display_text":"",
                "required_in_source": True
            }
        output[tablename]["fields"].append(field_attributes)

    # Write completed draft JSON as csv into the logging folder
    output_path = os.path.join(logging_path, (tablename+".json"))
    with open(output_path, "w") as results:
        json.dump(output, results, sort_keys=True, indent=2)

    # TODO: consider using logging.info to report success?
    print(tablename + " JSON table file created.")

def make_all_json(manifest_path):

    completed_tables = {}
    manifest = ManifestReader(path=manifest_path)
    if manifest.has_unique_ids():
        for manifest_row in manifest:
            tablename = manifest_row['destination_table']
            encoding = manifest_row.get('encoding')
            if tablename not in completed_tables:
                if manifest_row['include_flag'] == 'use':
                    filepath = os.path.abspath(
                            os.path.join(manifest_row['local_folder'],
                                        manifest_row['filepath']
                                        ))
                    make_draft_json(filepath, tablename, encoding)
                    completed_tables[tablename] = filepath
                    print("completed {}".format(tablename))
            else:
                print("skipping {}, already loaded".format(manifest_row['unique_data_id']))

    print("Finished! Used these files to generate json:")
    print(completed_tables)

def checkTable(table_name, table_info):
    """
    Check whether the new table is in table_info.json file

    :param: table_name - the new data to be added table_info.json
    :param type: string

    :param: table_info - path to the table_info.json file (currently called meta.json)
    :param type: string

    :return: boolean value
    : True - table_name found in table_info.json
    : False - table_name NOT found
    """
    if not(os.path.isfile(table_info)):
        raise ValueError("Path to table_info.json is invalid")

    # read the data from current json file
    with open(table_info, "r") as json_file:
        current_data = json_file.read()

    # decode the json file, result is a "dict", cast the result as a "set"
    current_tables = json.loads(current_data)
    table_set = set(current_tables.keys())

    return table_name in table_set

if __name__ == '__main__':
    # TODO: refactor to take in cmd line arguments for csv_filename, table_name,
    # TODO: and encoding

    if 'single' in sys.argv:
        # Edit these values before running!
        csv_filename = os.path.abspath("/Users/KweningJ/Documents/Computer "
                                       "Programming/housing-insights/data/raw/"
                                       "tax_assessment/opendata/20170505/"
                                       "DCHousing.csv")
        table_name = "dchousing"
        encoding = "latin1" #only used for opening w/ Pandas. Try utf-8 if latin1 doesn't work. Put the successful value into manifest.csv
        make_draft_json(csv_filename, table_name, encoding)

    if 'multi' in sys.argv:
        manifest_path = os.path.abspath(
            os.path.join(python_filepath, 'scripts', 'manifest.csv'))
        make_all_json(manifest_path)

    if 'single-add' in sys.argv:
        """
        An option to add a new data source to both create a table JSON file
        then check if the table has been already to the master JSON file, table_info.json

        It performs the same procedure as 'single', then performs the additional
        checks before appending the new table to table_info.json
        """
        # development case
        #json_filepath = python_filepath + "/scripts/meta.json"
        #print(os.path.isfile(json_filepath))

        #new_table = "building_permits"
        #print(checkTable(new_table, json_filepath))

        # Edit the follow parameters before running

        # Change csv_filename to the file path of the raw data file
        csv_filename = os.path.abspath("/Users/KweningJ/Documents/Computer "
                                       "Programming/housing-insights/data/raw/"
                                       "tax_assessment/opendata/20170505/"
                                       "DCHousing.csv")

        # Change table_name to the table being added
        table_name = "dchousing"
        #only used for opening w/ Pandas. Try utf-8 if latin1 doesn't work. Put the successful value into manifest.csv
        encoding = "latin1"

        make_draft_json(csv_filename, table_name, encoding)
<commit_msg>adding functionality to append new data source to master json file<commit_after>"""
Module creates draft JSON file(s) from given csv data sources that after user
review will be inserted into the meta.json (soon to be called table_info.json)

You can think of meta.json as the skeleton template for the data in tables
loaded into the database.
"""

import logging
import json
import sys
import os

import pandas as pandas

python_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                               os.pardir, os.pardir))
sys.path.append(python_filepath)

from housinginsights.ingestion.DataReader import ManifestReader

#configuration
#See /logs/example-logging.py for usage examples
logging_path = os.path.abspath(os.path.join(python_filepath, "logs"))
logging_filename = os.path.abspath(os.path.join(logging_path, "ingestion.log"))
logging.basicConfig(filename=logging_filename, level=logging.DEBUG)
#Pushes everything from the logger to the command line output as well
logging.getLogger().addHandler(logging.StreamHandler())

def _sql_name_clean(name):
    """
    Internal function to clean column names for the sql_name JSON field.

    :param name: column name for sql_nam JSON field
    :type name: str

    :return: cleaned up column name as lower case
    """
    # TODO: add code to make sure doesn't end or begin with underscore
    for item in ["-"," ","."]:
        if item in name:
            name = name.replace(item, "_")
    return name.lower()

def _pandas_to_sql_data_type(pandas_type_string):
    """
    An internal mapping function that returns the sql equivalent data type of a
    given pandas data type.

    :param pandas_type_string: pandas data type to look for
    :type pandas_type_string: str

    :return: sql equivalent data type
    """
    mapping = {
        'object': 'text',
        'int64': 'integer',
        'float64': 'decimal',
        'datetime64': 'timestamp'
    }
    # TODO: refactor to return key with default look up?
    try:
        sql_type = mapping[pandas_type_string]
    except KeyError:
        sql_type = 'text'
    return sql_type

def make_draft_json(filename, tablename, encoding):
    """
    Create a draft JSON file for the new table that will be reviewed and
    cleaned up before inserted into meta.json (soon to be called
    table_info.json).

    The draft output will be saved within '/housing-insights/python/logs'
    folder.

    :param filename: the filename of the csv data source
    :type filename: str

    :param tablename: the table name to be used in sql database
    :type tablename: str

    :param encoding: character encoding to pass to pandas - use 'latin1' or
    'utf-8' if that doesn't.
    :type encoding: str

    :return: None
    """

    # basic structure of the draft JSON output
    output = {
        tablename: {
            # TODO: change cleaner naming scheme to CamelCase
            "cleaner": tablename + "{}".format("_cleaner"),
            "replace_table": True,
            "fields": []
        }
    }

    # TODO: insert 'unique_data_id' field into draft JSON
    unique_data_id_field = {
        "display_name": "Unique data ID",
        "display_text": "Identifies which source file this record came from",
        "source_name": "unique_data_id",
        "sql_name": "unique_data_id",
        "type": "text",
        "required_in_source": False
    }
    output[tablename]["fields"].append(unique_data_id_field)

    # read csv file as pandas data frame and column headers
    dataframe_file = pandas.read_csv(filename, encoding=encoding)
    dataframe_iterator = dataframe_file.columns

    # Iterate through each field and populate draft JSON with attributes info.
    for field in dataframe_iterator:
        pandas_type = str(dataframe_file[field].dtypes)
        sql_type = _pandas_to_sql_data_type(pandas_type)

        field_attributes = {
                "type": sql_type,
                "source_name": field,
                "sql_name": _sql_name_clean(field),
                "display_name": _sql_name_clean(field),
                "display_text":"",
                "required_in_source": True
            }
        output[tablename]["fields"].append(field_attributes)

    # Write completed draft JSON as csv into the logging folder
    output_path = os.path.join(logging_path, (tablename+".json"))
    with open(output_path, "w") as results:
        json.dump(output, results, sort_keys=True, indent=2)

    # TODO: consider using logging.info to report success?
    print(tablename + " JSON table file created.")

def make_all_json(manifest_path):

    completed_tables = {}
    manifest = ManifestReader(path=manifest_path)
    if manifest.has_unique_ids():
        for manifest_row in manifest:
            tablename = manifest_row['destination_table']
            encoding = manifest_row.get('encoding')
            if tablename not in completed_tables:
                if manifest_row['include_flag'] == 'use':
                    filepath = os.path.abspath(
                            os.path.join(manifest_row['local_folder'],
                                        manifest_row['filepath']
                                        ))
                    make_draft_json(filepath, tablename, encoding)
                    completed_tables[tablename] = filepath
                    print("completed {}".format(tablename))
            else:
                print("skipping {}, already loaded".format(manifest_row['unique_data_id']))

    print("Finished! Used these files to generate json:")
    print(completed_tables)

def checkTable(table_name, table_info):
    """
    Check whether the new table is in table_info.json file

    :param: table_name - the new data to be added table_info.json
    :param type: string

    :param: table_info - path to the table_info.json file (currently called meta.json)
    :param type: string

    :return: boolean value
    : True - table_name found in table_info.json
    : False - table_name NOT found
    """
    if not(os.path.isfile(table_info)):
        raise ValueError("Path to table_info.json is invalid")

    # read the data from current json file
    with open(table_info, "r") as json_file:
        current_data = json_file.read()

    # decode the json file, result is a "dict", cast the result as a "set"
    current_tables = json.loads(current_data)
    table_set = set(current_tables.keys())

    return table_name in table_set

def appendJSON(new_json, master_json):
    """
    Copy the file content of new_json and append to the file content in meta_json

    :param:  new_json - path to the new json file
    :param type: string

    :param:  master_json - path to the table_info.json file
    :param type: string

    :return: void
    """
    if not(os.path.isfile(new_json) and os.path.isfile(master_json)):
        raise ValueError("Path to one of the JSON files is invalid")

    # Update the master JSON file, table_info.json
    new_data = json.loads(new_json)
    table_name = new_data.keys()[0]
    master_data = json.loads(master_json)
    master_json.update(new_data)
    json.dump(master_json, open(master_json, 'w'))
    print("%s appended to table_info.json file")

if __name__ == '__main__':
    # TODO: refactor to take in cmd line arguments for csv_filename, table_name,
    # TODO: and encoding

    if 'single' in sys.argv:
        # Edit these values before running!
        csv_filename = os.path.abspath("/Users/KweningJ/Documents/Computer "
                                       "Programming/housing-insights/data/raw/"
                                       "tax_assessment/opendata/20170505/"
                                       "DCHousing.csv")
        table_name = "dchousing"
        encoding = "latin1" #only used for opening w/ Pandas. Try utf-8 if latin1 doesn't work. Put the successful value into manifest.csv
        make_draft_json(csv_filename, table_name, encoding)

    if 'multi' in sys.argv:
        manifest_path = os.path.abspath(
            os.path.join(python_filepath, 'scripts', 'manifest.csv'))
        make_all_json(manifest_path)

    if 'single-add' in sys.argv:
        """
        An option to add a new data source to both create a table JSON file
        then check if the table has been already to the master JSON file, table_info.json

        It performs the same procedure as 'single', then performs the additional
        checks before appending the new table to table_info.json
        """
        # Edit the follow parameters before running
        # TODO: refactor to take in cmd line arguments for csv_filename, table_name, encoding

        # Change csv_filename to the file path of the raw data file
        csv_filename = os.path.abspath("C:/Devs/SongDev.csv")

        # Change table_name to the table being added
        table_name = "project_stemp"
        #only used for opening w/ Pandas. Try utf-8 if latin1 doesn't work. Put the successful value into manifest.csv
        encoding = "latin1"

        make_draft_json(csv_filename, table_name, encoding)

        new_json_path = os.path.join(logging_path, (table_name+".json"))

        #json_filepath = python_filepath + "/scripts/meta.json"
        json_filepath = python_filepath + "/scripts/meta_sample.json"

        #print(new_json_path)
        #print(os.path.isfile(json_filepath))

        try:
            if checkTable(table_name, json_filepath):
                # the new table is already in table_info.json
                print("table already in master json")

            else:
                # the new table will be appended
                print("adding new table")
                appendJSON(new_json_path, json_filepath)
                #pass

        except ValueError:
            pass

<|endoftext|>


<commit_before>""" Script downloading and unpacking Cumulus bundles for the host """

import logging
import logging.config
import os
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from ConfigParser import SafeConfigParser, NoOptionError

try:
    from boto import s3
except ImportError:
    print('Could not import boto. Try installing it with "pip install boto"')
    sys.exit(1)

CONFIG = SafeConfigParser()

if sys.platform in ['win32', 'cygwin']:
    CONFIG.read('C:\\cumulus\\conf\\metadata.conf')
else:
    CONFIG.read('/etc/cumulus/metadata.conf')

# Configure logging
LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_LOGGERs': False,
    'formatters': {
        'standard': {
            'format': (
                '%(asctime)s - cumulus-bundle-handler - '
                '%(levelname)s - %(message)s'
            )
        },
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'formatter': 'standard',
            'filename': '/var/log/cumulus-bundle-handler.log',
            'mode': 'a',
            'maxBytes': 10485760,  # 10 MB
            'backupCount': 5
        }
    },
    'loggers': {
        '': {
            'handlers': ['console', 'file'],
            'level': 'WARNING',
            'propagate': True
        },
        'cumulus_bundle_handler': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG',
            'propagate': False
        }
    }
}

# Change the log file path on Windows systems
if sys.platform in ['win32', 'cygwin']:
    if not os.path.exists('C:\\cumulus\\logs'):
        os.makedirs('C:\\cumulus\\logs')
    LOGGING_CONFIG['handlers']['file']['filename'] = \
        'C:\\cumulus\\logs\\cumulus-bundle-handler.log'

# Read log level from the metadata.conf
try:
    LOGGING_CONFIG['handlers']['console']['level'] = CONFIG.get(
        'metadata', 'log-level').upper()
    LOGGING_CONFIG['handlers']['file']['level'] = CONFIG.get(
        'metadata', 'log-level').upper()
except NoOptionError:
    pass

logging.config.dictConfig(LOGGING_CONFIG)
LOGGER = logging.getLogger('cumulus_bundle_handler')

def main():
    """ Main function """
    _run_init_scripts(kill=True, start=False, other=True)

    bundle_types = CONFIG.get('metadata', 'bundle-types').split(',')
    if not bundle_types:
        LOGGER.error('Missing "bundle-types" in metadata.conf')
        sys.exit(1)

    _remove_old_files()

    for bundle_type in bundle_types:
        _download_and_unpack_bundle(bundle_type)

    _run_init_scripts(kill=False, start=True, other=True)

    LOGGER.info("Done updating host")

def _download_and_unpack_bundle(bundle_type):
    """ Download the bundle from AWS S3

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    """
    key, compression = _get_key(bundle_type)

    # If the bundle does not exist
    if not key:
        LOGGER.error('No bundle found. Exiting.')
        sys.exit(1)

    bundle = tempfile.NamedTemporaryFile(
        suffix='.{}'.format(compression),
        delete=False)
    bundle.close()
    LOGGER.info("Downloading s3://{}/{} to {}".format(
        CONFIG.get('metadata', 'bundle-bucket'),
        key.name,
        bundle.name))
    key.get_contents_to_filename(bundle.name)

    extraction_path = _get_extraction_path(bundle_type)

    # Unpack the bundle
    LOGGER.info("Unpacking {}".format(bundle.name))
    if compression == 'tar.bz2':
        archive = tarfile.open(bundle.name, 'r:bz2')
        _store_bundle_files(archive.getnames(), extraction_path)
    elif compression == 'tar.gz':
        archive = tarfile.open(bundle.name, 'r:gz')
        _store_bundle_files(archive.getnames(), extraction_path)
    elif compression == 'zip':
        archive = zipfile.ZipFile(bundle.name, 'r')
        _store_bundle_files(archive.namelist(), extraction_path)
    else:
        logging.error('Unsupported compression format: "{}"'.format(
            compression))
        sys.exit(1)

    try:
        LOGGER.info('Unpacking {} to {}'.format(bundle.name, extraction_path))
        archive.extractall(extraction_path)
    finally:
        archive.close()

    # Remove the downloaded package
    LOGGER.info("Removing temporary file {}".format(bundle.name))
    os.remove(bundle.name)

def _get_extraction_path(bundle_type):
    """ Returns the path to where the bundle should be extracted

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    :returns: str -- Path
    """
    path = '/'
    if sys.platform in ['win32', 'cygwin']:
        path = 'C:\\'

    if not os.path.exists(path):
        LOGGER.debug('Created extraction path {}'.format(path))
        os.makedirs(path)

    try:
        bundle_paths = CONFIG.get('metadata', 'bundle-extraction-paths')\
            .split('\n')

        for line in bundle_paths:
            if not line:
                continue

            try:
                for_bundle_type, extraction_path = line.split('->')
                for_bundle_type = for_bundle_type.strip()
                extraction_path = extraction_path.strip()

                if for_bundle_type == bundle_type:
                    path = extraction_path

            except ValueError:
                LOGGER.error(
                    'Error parsing bundle-extraction-paths: {}'.format(line))
                sys.exit(1)

    except NoOptionError:
        pass

    LOGGER.debug('Determined bundle extraction path to {} for {}'.format(
        path, bundle_type))

    return path

def _get_key(bundle_type):
    """ Returns the bundle key

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    :returns: (boto.s3.key, str) -- (S3 key object, compression type)
    """
    LOGGER.debug("Connecting to AWS S3")
    connection = s3.connect_to_region(
        CONFIG.get('metadata', 'region'),
        aws_access_key_id=CONFIG.get('metadata', 'access-key-id'),
        aws_secret_access_key=CONFIG.get('metadata', 'secret-access-key'))

    # Get the relevant bucket
    bucket_name = CONFIG.get('metadata', 'bundle-bucket')
    LOGGER.debug('Using bucket {}'.format(bucket_name))
    bucket = connection.get_bucket(bucket_name)

    # Download the bundle
    for compression in ['tar.bz2', 'tar.gz', 'zip']:
        key_name = (
            '{env}/{version}/bundle-{env}-{version}-{bundle}.{comp}'.format(
                env=CONFIG.get('metadata', 'environment'),
                version=CONFIG.get('metadata', 'version'),
                bundle=bundle_type,
                comp=compression))
        LOGGER.debug('Looking for bundle {}'.format(key_name))
        key = bucket.get_key(key_name)

        # When we have found a key, don't look any more
        if key:
            LOGGER.debug('Found bundle: {}'.format(key_name))
            return (key, compression)
        LOGGER.debug('Bundle not found: {}'.format(key_name))

    return (None, None)

def _remove_old_files():
    """ Remove files from previous bundle """
    cache_file = '/var/local/cumulus-bundle-handler.cache'
    if sys.platform in ['win32', 'cygwin']:
        if not os.path.exists('C:\\cumulus\\cache'):
            os.makedirs('C:\\cumulus\\cache')
        cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'

    if not os.path.exists(cache_file):
        LOGGER.info('No previous bundle files to clean up')
        return

    LOGGER.info('Removing old files and directories')

    with open(cache_file, 'r') as file_handle:
        for line in file_handle.readlines():
            line = line.replace('\n', '')

            if not os.path.exists(line):
                continue

            if os.path.isdir(line):
                try:
                    os.removedirs(line)
                    LOGGER.debug('Removing directory {}'.format(line))
                except OSError:
                    pass
            elif os.path.isfile(line):
                LOGGER.debug('Removing file {}'.format(line))
                os.remove(line)

                try:
                    os.removedirs(os.path.dirname(line))
                except OSError:
                    pass
            elif os.path.islink(line):
                LOGGER.debug('Removing link {}'.format(line))
                os.remove(line)

                try:
                    os.removedirs(os.path.dirname(line))
                except OSError:
                    pass
            else:
                LOGGER.warning('Unknown file type {}'.format(line))

    # Remove the cache file when done
    os.remove(cache_file)

def _run_command(command):
    """ Run arbitary command

    :type command: str
    :param command: Command to execute
    """
    LOGGER.info('Executing command: {}'.format(command))

    cmd = subprocess.Popen(
        command,
        shell=True,
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE)

    stdout, stderr = cmd.communicate()
    if stdout:
        print(stdout)
    if stderr:
        print(stderr)

    if cmd.returncode != 0:
        LOGGER.error('Command "{}" returned non-zero exit code {}'.format(
            command,
            cmd.returncode))
        sys.exit(cmd.returncode)

def _run_init_scripts(start=False, kill=False, other=False):
    """ Execute scripts in /etc/cumulus-init.d or C:\\cumulus\\init.d

    :type start: bool
    :param start: Run scripts starting with S
    :type kill: bool
    :param kill: Run scripts starting with K
    :type others: bool
    :param others: Run scripts not starting with S or K
    """
    init_dir = '/etc/cumulus-init.d'
    if sys.platform in ['win32', 'cygwin']:
        init_dir = 'C:\\cumulus\\init.d'

    # Run the post install scripts provided by the bundle
    if not os.path.exists(init_dir):
        LOGGER.info('No init scripts found in {}'.format(init_dir))
        return

    LOGGER.info('Running init scripts from {}'.format(init_dir))

    filenames = []
    for filename in os.listdir(init_dir):
        if os.path.isfile(os.path.join(init_dir, filename)):
            filenames.append(os.path.join(init_dir, filename))

    if start:
        for filename in filenames:
            if os.path.basename(filename)[0] == 'S':
                _run_command(os.path.abspath(filename))

    if kill:
        for filename in filenames:
            if os.path.basename(filename)[0] == 'K':
                _run_command(os.path.abspath(filename))

    if other:
        for filename in filenames:
            if os.path.basename(filename)[0] not in ['K', 'S']:
                _run_command(os.path.abspath(filename))

def _store_bundle_files(filenames, extraction_path):
    """ Store a list of bundle paths

    :type filenames: list
    :param filenames: List of full paths for all paths in the bundle'
    :type extraction_path: str
    :param extraction_path: Path to prefix all filenames with
    """
    cache_file = '/var/local/cumulus-bundle-handler.cache'
    if sys.platform in ['win32', 'cygwin']:
        if not os.path.exists('C:\\cumulus\\cache'):
            os.makedirs('C:\\cumulus\\cache')
        cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'

    file_handle = open(cache_file, 'a')
    try:
        for filename in filenames:
            if not filename:
                continue

            if sys.platform in ['win32', 'cygwin']:
                filename = '{}\\{}'.format(extraction_path, filename)
            else:
                filename = '{}/{}'.format(extraction_path, filename)

            file_handle.write('{}\n'.format(filename))

        LOGGER.debug('Stored bundle information in {}'.format(cache_file))
    finally:
        file_handle.close()
<commit_msg>Catching exception when config file is missing #101<commit_after>""" Script downloading and unpacking Cumulus bundles for the host """

import logging
import logging.config
import os
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from ConfigParser import SafeConfigParser, NoOptionError

try:
    from boto import s3
except ImportError:
    print('Could not import boto. Try installing it with "pip install boto"')
    sys.exit(1)

CONFIG = SafeConfigParser()

if sys.platform in ['win32', 'cygwin']:
    CONFIG_PATH = 'C:\\cumulus\\conf\\metadata.conf'
else:
    CONFIG_PATH = '/etc/cumulus/metadata.conf'

if not os.path(CONFIG_PATH):
    print('Error: Configuration file not found: {}'.format(CONFIG_PATH))
    sys.exit(1)

CONFIG.read(CONFIG_PATH)

# Configure logging
LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_LOGGERs': False,
    'formatters': {
        'standard': {
            'format': (
                '%(asctime)s - cumulus-bundle-handler - '
                '%(levelname)s - %(message)s'
            )
        },
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'formatter': 'standard',
            'filename': '/var/log/cumulus-bundle-handler.log',
            'mode': 'a',
            'maxBytes': 10485760,  # 10 MB
            'backupCount': 5
        }
    },
    'loggers': {
        '': {
            'handlers': ['console', 'file'],
            'level': 'WARNING',
            'propagate': True
        },
        'cumulus_bundle_handler': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG',
            'propagate': False
        }
    }
}

# Change the log file path on Windows systems
if sys.platform in ['win32', 'cygwin']:
    if not os.path.exists('C:\\cumulus\\logs'):
        os.makedirs('C:\\cumulus\\logs')
    LOGGING_CONFIG['handlers']['file']['filename'] = \
        'C:\\cumulus\\logs\\cumulus-bundle-handler.log'

# Read log level from the metadata.conf
try:
    LOGGING_CONFIG['handlers']['console']['level'] = CONFIG.get(
        'metadata', 'log-level').upper()
    LOGGING_CONFIG['handlers']['file']['level'] = CONFIG.get(
        'metadata', 'log-level').upper()
except NoOptionError:
    pass

logging.config.dictConfig(LOGGING_CONFIG)
LOGGER = logging.getLogger('cumulus_bundle_handler')

def main():
    """ Main function """
    _run_init_scripts(kill=True, start=False, other=True)

    bundle_types = CONFIG.get('metadata', 'bundle-types').split(',')
    if not bundle_types:
        LOGGER.error('Missing "bundle-types" in metadata.conf')
        sys.exit(1)

    _remove_old_files()

    for bundle_type in bundle_types:
        _download_and_unpack_bundle(bundle_type)

    _run_init_scripts(kill=False, start=True, other=True)

    LOGGER.info("Done updating host")

def _download_and_unpack_bundle(bundle_type):
    """ Download the bundle from AWS S3

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    """
    key, compression = _get_key(bundle_type)

    # If the bundle does not exist
    if not key:
        LOGGER.error('No bundle found. Exiting.')
        sys.exit(1)

    bundle = tempfile.NamedTemporaryFile(
        suffix='.{}'.format(compression),
        delete=False)
    bundle.close()
    LOGGER.info("Downloading s3://{}/{} to {}".format(
        CONFIG.get('metadata', 'bundle-bucket'),
        key.name,
        bundle.name))
    key.get_contents_to_filename(bundle.name)

    extraction_path = _get_extraction_path(bundle_type)

    # Unpack the bundle
    LOGGER.info("Unpacking {}".format(bundle.name))
    if compression == 'tar.bz2':
        archive = tarfile.open(bundle.name, 'r:bz2')
        _store_bundle_files(archive.getnames(), extraction_path)
    elif compression == 'tar.gz':
        archive = tarfile.open(bundle.name, 'r:gz')
        _store_bundle_files(archive.getnames(), extraction_path)
    elif compression == 'zip':
        archive = zipfile.ZipFile(bundle.name, 'r')
        _store_bundle_files(archive.namelist(), extraction_path)
    else:
        logging.error('Unsupported compression format: "{}"'.format(
            compression))
        sys.exit(1)

    try:
        LOGGER.info('Unpacking {} to {}'.format(bundle.name, extraction_path))
        archive.extractall(extraction_path)
    finally:
        archive.close()

    # Remove the downloaded package
    LOGGER.info("Removing temporary file {}".format(bundle.name))
    os.remove(bundle.name)

def _get_extraction_path(bundle_type):
    """ Returns the path to where the bundle should be extracted

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    :returns: str -- Path
    """
    path = '/'
    if sys.platform in ['win32', 'cygwin']:
        path = 'C:\\'

    if not os.path.exists(path):
        LOGGER.debug('Created extraction path {}'.format(path))
        os.makedirs(path)

    try:
        bundle_paths = CONFIG.get('metadata', 'bundle-extraction-paths')\
            .split('\n')

        for line in bundle_paths:
            if not line:
                continue

            try:
                for_bundle_type, extraction_path = line.split('->')
                for_bundle_type = for_bundle_type.strip()
                extraction_path = extraction_path.strip()

                if for_bundle_type == bundle_type:
                    path = extraction_path

            except ValueError:
                LOGGER.error(
                    'Error parsing bundle-extraction-paths: {}'.format(line))
                sys.exit(1)

    except NoOptionError:
        pass

    LOGGER.debug('Determined bundle extraction path to {} for {}'.format(
        path, bundle_type))

    return path

def _get_key(bundle_type):
    """ Returns the bundle key

    :type bundle_type: str
    :param bundle_type: Bundle type to download
    :returns: (boto.s3.key, str) -- (S3 key object, compression type)
    """
    LOGGER.debug("Connecting to AWS S3")
    connection = s3.connect_to_region(
        CONFIG.get('metadata', 'region'),
        aws_access_key_id=CONFIG.get('metadata', 'access-key-id'),
        aws_secret_access_key=CONFIG.get('metadata', 'secret-access-key'))

    # Get the relevant bucket
    bucket_name = CONFIG.get('metadata', 'bundle-bucket')
    LOGGER.debug('Using bucket {}'.format(bucket_name))
    bucket = connection.get_bucket(bucket_name)

    # Download the bundle
    for compression in ['tar.bz2', 'tar.gz', 'zip']:
        key_name = (
            '{env}/{version}/bundle-{env}-{version}-{bundle}.{comp}'.format(
                env=CONFIG.get('metadata', 'environment'),
                version=CONFIG.get('metadata', 'version'),
                bundle=bundle_type,
                comp=compression))
        LOGGER.debug('Looking for bundle {}'.format(key_name))
        key = bucket.get_key(key_name)

        # When we have found a key, don't look any more
        if key:
            LOGGER.debug('Found bundle: {}'.format(key_name))
            return (key, compression)
        LOGGER.debug('Bundle not found: {}'.format(key_name))

    return (None, None)

def _remove_old_files():
    """ Remove files from previous bundle """
    cache_file = '/var/local/cumulus-bundle-handler.cache'
    if sys.platform in ['win32', 'cygwin']:
        if not os.path.exists('C:\\cumulus\\cache'):
            os.makedirs('C:\\cumulus\\cache')
        cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'

    if not os.path.exists(cache_file):
        LOGGER.info('No previous bundle files to clean up')
        return

    LOGGER.info('Removing old files and directories')

    with open(cache_file, 'r') as file_handle:
        for line in file_handle.readlines():
            line = line.replace('\n', '')

            if not os.path.exists(line):
                continue

            if os.path.isdir(line):
                try:
                    os.removedirs(line)
                    LOGGER.debug('Removing directory {}'.format(line))
                except OSError:
                    pass
            elif os.path.isfile(line):
                LOGGER.debug('Removing file {}'.format(line))
                os.remove(line)

                try:
                    os.removedirs(os.path.dirname(line))
                except OSError:
                    pass
            elif os.path.islink(line):
                LOGGER.debug('Removing link {}'.format(line))
                os.remove(line)

                try:
                    os.removedirs(os.path.dirname(line))
                except OSError:
                    pass
            else:
                LOGGER.warning('Unknown file type {}'.format(line))

    # Remove the cache file when done
    os.remove(cache_file)

def _run_command(command):
    """ Run arbitary command

    :type command: str
    :param command: Command to execute
    """
    LOGGER.info('Executing command: {}'.format(command))

    cmd = subprocess.Popen(
        command,
        shell=True,
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE)

    stdout, stderr = cmd.communicate()
    if stdout:
        print(stdout)
    if stderr:
        print(stderr)

    if cmd.returncode != 0:
        LOGGER.error('Command "{}" returned non-zero exit code {}'.format(
            command,
            cmd.returncode))
        sys.exit(cmd.returncode)

def _run_init_scripts(start=False, kill=False, other=False):
    """ Execute scripts in /etc/cumulus-init.d or C:\\cumulus\\init.d

    :type start: bool
    :param start: Run scripts starting with S
    :type kill: bool
    :param kill: Run scripts starting with K
    :type others: bool
    :param others: Run scripts not starting with S or K
    """
    init_dir = '/etc/cumulus-init.d'
    if sys.platform in ['win32', 'cygwin']:
        init_dir = 'C:\\cumulus\\init.d'

    # Run the post install scripts provided by the bundle
    if not os.path.exists(init_dir):
        LOGGER.info('No init scripts found in {}'.format(init_dir))
        return

    LOGGER.info('Running init scripts from {}'.format(init_dir))

    filenames = []
    for filename in os.listdir(init_dir):
        if os.path.isfile(os.path.join(init_dir, filename)):
            filenames.append(os.path.join(init_dir, filename))

    if start:
        for filename in filenames:
            if os.path.basename(filename)[0] == 'S':
                _run_command(os.path.abspath(filename))

    if kill:
        for filename in filenames:
            if os.path.basename(filename)[0] == 'K':
                _run_command(os.path.abspath(filename))

    if other:
        for filename in filenames:
            if os.path.basename(filename)[0] not in ['K', 'S']:
                _run_command(os.path.abspath(filename))

def _store_bundle_files(filenames, extraction_path):
    """ Store a list of bundle paths

    :type filenames: list
    :param filenames: List of full paths for all paths in the bundle'
    :type extraction_path: str
    :param extraction_path: Path to prefix all filenames with
    """
    cache_file = '/var/local/cumulus-bundle-handler.cache'
    if sys.platform in ['win32', 'cygwin']:
        if not os.path.exists('C:\\cumulus\\cache'):
            os.makedirs('C:\\cumulus\\cache')
        cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'

    file_handle = open(cache_file, 'a')
    try:
        for filename in filenames:
            if not filename:
                continue

            if sys.platform in ['win32', 'cygwin']:
                filename = '{}\\{}'.format(extraction_path, filename)
            else:
                filename = '{}/{}'.format(extraction_path, filename)

            file_handle.write('{}\n'.format(filename))

        LOGGER.debug('Stored bundle information in {}'.format(cache_file))
    finally:
        file_handle.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import imp

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def which(program):
    import os

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

def check_python_modules():
    print("\033[1mCheck modules\033[0m")
    required_modules = ['argparse', 'matplotlib', 'natsort', 'MySQLdb',
                        'cPickle', 'detl', 'theano', 'dropbox',
                        'webbrowser', 'hashlib', 'shapely', 'numpy']
    found = []
    for required_module in required_modules:
        try:
            imp.find_module(required_module)
            check = "module '%s' ... %sfound%s" % (required_module,
                                                   bcolors.OKGREEN,
                                                   bcolors.ENDC)
            print(check)
            found.append(required_module)
        except ImportError:
            print("module '%s' ... %sNOT%s found" % (required_module,
                                                     bcolors.WARNING,
                                                     bcolors.ENDC))

    if "argparse" in found:
        import argparse
        print("argparse version: %s (1.1 tested)" % argparse.__version__)
    if "matplotlib" in found:
        import matplotlib
        print("matplotlib version: %s (1.2.1 tested)" % matplotlib.__version__)
    if "natsort" in found:
        import natsort
        print("natsort version: %s (3.4.0 tested, 3.4.0 > required)" %
              natsort.__version__)
    if "MySQLdb" in found:
        import MySQLdb
        print("MySQLdb version: %s (1.2.3 tested)" %
              MySQLdb.__version__)
    if "theano" in found:
        import theano
        print("theano version: %s (0.6.0 tested)" %
              theano.__version__)
    if "shapely" in found:
        import shapely
        print("shapely version: %s (1.2.14 tested)" %
              shapely.__version__)
    if "numpy" in found:
        import numpy
        print("numpy version: %s (1.8.1 tested)" %
              numpy.__version__)
    if "cPickle" in found:
        import cPickle
        print("cPickle version: %s (1.71 tested)" %
              cPickle.__version__)
        print("cPickle HIGHEST_PROTOCOL: %s (2 required)" %
              cPickle.HIGHEST_PROTOCOL)

def check_executables():
    print("\033[1mCheck executables\033[0m")
    required_executables = ["pfile_create"]
    for executable in required_executables:
        path = which(executable)
        if path is None:
            print("%s ... %sNOT%s found" % (executable, bcolors.WARNING,
                                            bcolors.ENDC))
        else:
            print("%s ... %sfound%s at %s" % (executable, bcolors.OKGREEN,
                                              bcolors.ENDC, path))

def main():
    check_python_modules()
    check_executables()

if __name__ == '__main__':
    main()
<commit_msg>added python version requirement<commit_after>#!/usr/bin/env python

import imp
import sys
import platform

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def which(program):
    import os

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None

def check_python_version():
    # Required due to multiple with statements on one line
    req_version = (2, 7)
    cur_version = sys.version_info
    if cur_version >= req_version:
        print("Python version... %sOK%s (found %s, requires %s)" %
              (bcolors.OKGREEN, bcolors.ENDC, str(platform.python_version()),
               str(req_version[0]) + "." + str(req_version[1])))
    else:
        print("Python version... %sFAIL%s (found %s, requires %s)" %
              (bcolors.OKGREEN, bcolors.ENDC, str(cur_version),
               str(req_version)))

def check_python_modules():
    print("\033[1mCheck modules\033[0m")
    required_modules = ['argparse', 'matplotlib', 'natsort', 'MySQLdb',
                        'cPickle', 'detl', 'theano', 'dropbox',
                        'webbrowser', 'hashlib', 'shapely', 'numpy', 'wget']
    found = []
    for required_module in required_modules:
        try:
            imp.find_module(required_module)
            check = "module '%s' ... %sfound%s" % (required_module,
                                                   bcolors.OKGREEN,
                                                   bcolors.ENDC)
            print(check)
            found.append(required_module)
        except ImportError:
            print("module '%s' ... %sNOT%s found" % (required_module,
                                                     bcolors.WARNING,
                                                     bcolors.ENDC))

    if "argparse" in found:
        import argparse
        print("argparse version: %s (1.1 tested)" % argparse.__version__)
    if "matplotlib" in found:
        import matplotlib
        print("matplotlib version: %s (1.2.1 tested)" % matplotlib.__version__)
    if "natsort" in found:
        import natsort
        print("natsort version: %s (3.4.0 tested, 3.4.0 > required)" %
              natsort.__version__)
    if "MySQLdb" in found:
        import MySQLdb
        print("MySQLdb version: %s (1.2.3 tested)" %
              MySQLdb.__version__)
    if "theano" in found:
        import theano
        print("theano version: %s (0.6.0 tested)" %
              theano.__version__)
    if "shapely" in found:
        import shapely
        print("shapely version: %s (1.2.14 tested)" %
              shapely.__version__)
    if "numpy" in found:
        import numpy
        print("numpy version: %s (1.8.1 tested)" %
              numpy.__version__)
    if "wget" in found:
        import wget
        print("wget version: %s (2.2 tested)" %
              wget.__version__)
    if "cPickle" in found:
        import cPickle
        print("cPickle version: %s (1.71 tested)" %
              cPickle.__version__)
        print("cPickle HIGHEST_PROTOCOL: %s (2 required)" %
              cPickle.HIGHEST_PROTOCOL)

def check_executables():
    print("\033[1mCheck executables\033[0m")
    required_executables = ["pfile_create"]
    for executable in required_executables:
        path = which(executable)
        if path is None:
            print("%s ... %sNOT%s found" % (executable, bcolors.WARNING,
                                            bcolors.ENDC))
        else:
            print("%s ... %sfound%s at %s" % (executable, bcolors.OKGREEN,
                                              bcolors.ENDC, path))

def main():
    check_python_version()
    check_python_modules()
    check_executables()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
homeassistant.components.alarm_control_panel.mqtt

This platform enables the possibility to control a MQTT alarm.
In this platform, 'state_topic' and 'command_topic' are required.
The alarm will only change state after receiving the a new state
from 'state_topic'. If these messages are published with RETAIN flag,
the MQTT alarm will receive an instant state update after subscription
and will start with correct state. Otherwise, the initial state will
be 'unknown'.

Configuration:

alarm_control_panel:
  platform: mqtt
  name: "MQTT Alarm"
  state_topic: "home/alarm"
  command_topic: "home/alarm/set"
  qos: 0
  payload_disarm: "DISARM"
  payload_arm_home: "ARM_HOME"
  payload_arm_away: "ARM_AWAY"
  code: "mySecretCode"

Variables:

name
*Optional
The name of the alarm. Default is 'MQTT Alarm'.

state_topic
*Required
The MQTT topic subscribed to receive state updates.

command_topic
*Required
The MQTT topic to publish commands to change the alarm state.

qos
*Optional
The maximum QoS level of the state topic. Default is 0.
This QoS will also be used to publishing messages.

payload_disarm
*Optional
The payload do disarm alarm. Default is "DISARM".

payload_arm_home
*Optional
The payload to set armed-home mode. Default is "ARM_HOME".

payload_arm_away
*Optional
The payload to set armed-away mode. Default is "ARM_AWAY".

code
*Optional
If defined, specifies a code to enable or disable the alarm in the frontend.

"""

import logging
import homeassistant.components.mqtt as mqtt
import homeassistant.components.alarm_control_panel as alarm

from homeassistant.const import (STATE_UNKNOWN)

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = "MQTT Alarm"
DEFAULT_QOS = 0
DEFAULT_PAYLOAD_DISARM = "DISARM"
DEFAULT_PAYLOAD_ARM_HOME = "ARM_HOME"
DEFAULT_PAYLOAD_ARM_AWAY = "ARM_AWAY"

DEPENDENCIES = ['mqtt']

def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Sets up the MQTT platform. """

    if config.get('state_topic') is None:
        _LOGGER.error("Missing required variable: state_topic")
        return False

    if config.get('command_topic') is None:
        _LOGGER.error("Missing required variable: command_topic")
        return False

    add_devices([MqttAlarm(
        hass,
        config.get('name', DEFAULT_NAME),
        config.get('state_topic'),
        config.get('command_topic'),
        config.get('qos', DEFAULT_QOS),
        config.get('payload_disarm', DEFAULT_PAYLOAD_DISARM),
        config.get('payload_arm_home', DEFAULT_PAYLOAD_ARM_HOME),
        config.get('payload_arm_away', DEFAULT_PAYLOAD_ARM_AWAY),
        config.get('code'))])

# pylint: disable=too-many-arguments, too-many-instance-attributes
class MqttAlarm(alarm.AlarmControlPanel):
    """ represents a MQTT alarm status within home assistant. """

    def __init__(self, hass, name, state_topic, command_topic, qos,
                 payload_disarm, payload_arm_home, payload_arm_away, code):
        self._state = STATE_UNKNOWN
        self._hass = hass
        self._name = name
        self._state_topic = state_topic
        self._command_topic = command_topic
        self._qos = qos
        self._payload_disarm = payload_disarm
        self._payload_arm_home = payload_arm_home
        self._payload_arm_away = payload_arm_away
        self._code = code

        def message_received(topic, payload, qos):
            """ A new MQTT message has been received. """
            self._state = payload
            self.update_ha_state()

        mqtt.subscribe(hass, self._state_topic, message_received, self._qos)

    @property
    def should_poll(self):
        """ No polling needed """
        return False

    @property
    def name(self):
        """ Returns the name of the device. """
        return self._name

    @property
    def state(self):
        """ Returns the state of the device. """
        return self._state

    @property
    def code_format(self):
        """ One or more characters if code is defined """
        return None if self._code is None else '.+'

    def alarm_disarm(self, code=None):
        """ Send disarm command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_disarm, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while disarming!")

    def alarm_arm_home(self, code=None):
        """ Send arm home command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_arm_home, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while arming home!")

    def alarm_arm_away(self, code=None):
        """ Send arm away command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_arm_away, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while arming away!")
<commit_msg>Update docstring<commit_after>"""
homeassistant.components.alarm_control_panel.mqtt
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

This platform enables the possibility to control a MQTT alarm.
In this platform, 'state_topic' and 'command_topic' are required.
The alarm will only change state after receiving the a new state
from 'state_topic'. If these messages are published with RETAIN flag,
the MQTT alarm will receive an instant state update after subscription
and will start with correct state. Otherwise, the initial state will
be 'unknown'.

Configuration:

alarm_control_panel:
  platform: mqtt
  name: "MQTT Alarm"
  state_topic: "home/alarm"
  command_topic: "home/alarm/set"
  qos: 0
  payload_disarm: "DISARM"
  payload_arm_home: "ARM_HOME"
  payload_arm_away: "ARM_AWAY"
  code: "mySecretCode"

Variables:

name
*Optional
The name of the alarm. Default is 'MQTT Alarm'.

state_topic
*Required
The MQTT topic subscribed to receive state updates.

command_topic
*Required
The MQTT topic to publish commands to change the alarm state.

qos
*Optional
The maximum QoS level of the state topic. Default is 0.
This QoS will also be used to publishing messages.

payload_disarm
*Optional
The payload do disarm alarm. Default is "DISARM".

payload_arm_home
*Optional
The payload to set armed-home mode. Default is "ARM_HOME".

payload_arm_away
*Optional
The payload to set armed-away mode. Default is "ARM_AWAY".

code
*Optional
If defined, specifies a code to enable or disable the alarm in the frontend.
"""
import logging
import homeassistant.components.mqtt as mqtt
import homeassistant.components.alarm_control_panel as alarm

from homeassistant.const import (STATE_UNKNOWN)

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = "MQTT Alarm"
DEFAULT_QOS = 0
DEFAULT_PAYLOAD_DISARM = "DISARM"
DEFAULT_PAYLOAD_ARM_HOME = "ARM_HOME"
DEFAULT_PAYLOAD_ARM_AWAY = "ARM_AWAY"

DEPENDENCIES = ['mqtt']

def setup_platform(hass, config, add_devices, discovery_info=None):
    """ Sets up the MQTT platform. """

    if config.get('state_topic') is None:
        _LOGGER.error("Missing required variable: state_topic")
        return False

    if config.get('command_topic') is None:
        _LOGGER.error("Missing required variable: command_topic")
        return False

    add_devices([MqttAlarm(
        hass,
        config.get('name', DEFAULT_NAME),
        config.get('state_topic'),
        config.get('command_topic'),
        config.get('qos', DEFAULT_QOS),
        config.get('payload_disarm', DEFAULT_PAYLOAD_DISARM),
        config.get('payload_arm_home', DEFAULT_PAYLOAD_ARM_HOME),
        config.get('payload_arm_away', DEFAULT_PAYLOAD_ARM_AWAY),
        config.get('code'))])

# pylint: disable=too-many-arguments, too-many-instance-attributes
class MqttAlarm(alarm.AlarmControlPanel):
    """ represents a MQTT alarm status within home assistant. """

    def __init__(self, hass, name, state_topic, command_topic, qos,
                 payload_disarm, payload_arm_home, payload_arm_away, code):
        self._state = STATE_UNKNOWN
        self._hass = hass
        self._name = name
        self._state_topic = state_topic
        self._command_topic = command_topic
        self._qos = qos
        self._payload_disarm = payload_disarm
        self._payload_arm_home = payload_arm_home
        self._payload_arm_away = payload_arm_away
        self._code = code

        def message_received(topic, payload, qos):
            """ A new MQTT message has been received. """
            self._state = payload
            self.update_ha_state()

        mqtt.subscribe(hass, self._state_topic, message_received, self._qos)

    @property
    def should_poll(self):
        """ No polling needed """
        return False

    @property
    def name(self):
        """ Returns the name of the device. """
        return self._name

    @property
    def state(self):
        """ Returns the state of the device. """
        return self._state

    @property
    def code_format(self):
        """ One or more characters if code is defined """
        return None if self._code is None else '.+'

    def alarm_disarm(self, code=None):
        """ Send disarm command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_disarm, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while disarming!")

    def alarm_arm_home(self, code=None):
        """ Send arm home command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_arm_home, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while arming home!")

    def alarm_arm_away(self, code=None):
        """ Send arm away command. """
        if code == str(self._code) or self.code_format is None:
            mqtt.publish(self.hass, self._command_topic,
                         self._payload_arm_away, self._qos)
        else:
            _LOGGER.warning("Wrong code entered while arming away!")
<|endoftext|>


<commit_before>"""
Copyright (c) 2014, CloudSigma AG
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the CloudSigma AG nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL CloudSigma AG BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
__version__ = "0.1.6"
<commit_msg> Bump version to 0.2.0<commit_after>"""
Copyright (c) 2014, CloudSigma AG
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the CloudSigma AG nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL CloudSigma AG BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
__version__ = "0.2.0"
<|endoftext|>


<commit_before>import datetime

import netsvc
from openerp.osv import orm, fields
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT
from tools.translate import _

class OrderRequirementLineAdd(orm.TransientModel):
    _name = "order.requirement.line.add"
    _description = "Split in Production lots"

    def default_get(self, cr, uid, fields, context=None):
        if context is None:
            context = {}
        res = super(OrderRequirementLineAdd, self).default_get(cr, uid, fields, context=context)
        if context.get('active_id'):
            line = self.pool['order.requirement.line'].browse(cr, uid, context['active_id'], context=context)
            if 'order_id' in fields:
                order_id = line.order_requirement_id.sale_order_id
                res.update(order_id=order_id.id)
        #     if 'product_id' in fields:
        #         res.update({'product_id': move.product_id.id})
        #     if 'product_uom' in fields:
        #         res.update({'product_uom': move.product_uom.id})
        #     if 'qty' in fields:
        #         res.update({'qty': move.product_qty})
        #     if 'use_exist' in fields:
        #         res.update({'use_exist': (move.picking_id and move.picking_id.type=='out' and True) or False})
        #     if 'location_id' in fields:
        #         res.update({'location_id': move.location_id.id})
        return res

    def _get_order_line(self, cr, uid, context=None):
        sale_order_line_obj = self.pool['sale.order.line']

        line = self.pool['order.requirement'].browse(cr, uid, context['order_id'], context=context)
        order_id = line.sale_order_id.id

        sale_order_line_ids = sale_order_line_obj.search(cr, uid, [('order_id', '=', order_id)], context=context)
        res = sale_order_line_obj.name_get(cr, uid, sale_order_line_ids, context=context)

        return res

    _columns = {
        'order_id': fields.many2one('sale.order', 'Order', required=True, select=True),
        'order_line': fields.selection(_get_order_line, 'Order Line', required=False),
        # 'order_line_id': fields.many2one('sale.order.line', 'Production Lots'),
        # 'order_line_ids': fields.one2many('stock.move.split.lines', 'wizard_id', 'Production Lots'),
    }

    def link(self, cr, uid, ids, context=None):
        context = context or self.pool['res.users'].context_get(cr, uid)
        res = self._link(cr, uid, ids, context.get('active_ids'), context=context)
        return {'type': 'ir.actions.act_window_close'}

    def _link(self, cr, uid, ids, line_ids, context=None):
        context = context or self.pool['res.users'].context_get(cr, uid)
        for wizard in self.browse(cr, uid, ids, context):
            if wizard.order_line:
                line_id = int(wizard.order_line)
                order_requirement_line = self.pool['order.requirement.line'].browse(cr, uid, line_ids[0], context)
                order_requirement_line.write({'sale_order_line_id': line_id})

                order_requirement = self.pool['order.requirement'].browse(cr, uid, context['order_id'], context)

                # create picking
                pick_type = 'internal'
                ir_sequence_obj = self.pool['ir.sequence']
                stock_picking_obj = self.pool['stock.picking']

                pick_name = ir_sequence_obj.get(cr, uid, 'stock.picking.' + pick_type)
                order = order_requirement.sale_order_id
                date_planned = datetime.date.today().strftime(DEFAULT_SERVER_DATE_FORMAT)

                picking_vals = {
                    'name': pick_name,
                    'origin': _('Order Requirement') + ' ' + order.name,
                    'date': date_planned,
                    'type': pick_type,
                    'state': 'auto',
                    'move_type': 'one',
                    'sale_id': order.id,
                    'address_id': order.partner_shipping_id.id,
                    'note': order.note,
                    'invoice_state': 'none',
                    'company_id': order.company_id.id,
                    'auto_picking': True,
                }
                if order.project_project:
                    project = order.project_project
                    picking_vals.update({
                        'project_id': project.id,
                        'account_id': project.analytic_account_id.id,
                        'sale_project': project.id
                    })

                picking_id = stock_picking_obj.create(cr, uid, picking_vals, context)

                location_id = order.shop_id.warehouse_id.lot_stock_id.id
                output_id = order.shop_id.warehouse_id.lot_output_id.id
                price_unit = 0.0

                if order_requirement_line.qty != 0.0:
                    price_unit = order_requirement_line.product_id.cost_price

                move_vals = {
                    'name': order_requirement_line.product_id.name[:250],
                    'picking_id': picking_id,
                    'product_id': order_requirement_line.product_id.id,
                    'date': date_planned,
                    'date_expected': date_planned,
                    'product_qty': order_requirement_line.qty,
                    'product_uom': order_requirement_line.product_id.uom_id.id,
                    # 'product_uos_qty': (line.product_uos and line.product_uos_qty) or line.product_uom_qty,
                    # 'product_uos': (line.product_uos and line.product_uos.id) \
                    #                or line.product_uom.id,
                    # 'product_packaging': line.product_packaging.id,
                    # 'address_id': order.partner_shipping_id.id,
                    'location_id': location_id,
                    'location_dest_id': output_id,
                    'sale_line_id': line_id,
                    'tracking_id': False,
                    'state': 'draft',
                    # 'state': 'waiting',
                    'company_id': order.company_id.id,
                    'price_unit': price_unit
                }

                move_id = self.pool['stock.move'].create(cr, uid, move_vals, context)
                wf_service = netsvc.LocalService("workflow")
                wf_service.trg_validate(uid, 'stock.picking', picking_id, 'button_confirm', cr)
                stock_picking_obj.force_assign(cr, uid, [picking_id], context)
                ctx = context.copy()
                ctx['force_commit'] = True
                stock_picking_obj._commit_cost(cr, uid, [picking_id], ctx)

        return True
<commit_msg>[sale_order_requirement] fix error on missing context<commit_after>import datetime

import netsvc
from openerp.osv import orm, fields
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT
from tools.translate import _

class OrderRequirementLineAdd(orm.TransientModel):
    _name = "order.requirement.line.add"
    _description = "Split in Production lots"

    def default_get(self, cr, uid, fields, context=None):
        if context is None:
            context = {}
        res = super(OrderRequirementLineAdd, self).default_get(cr, uid, fields, context=context)
        if context.get('active_id'):
            line = self.pool['order.requirement.line'].browse(cr, uid, context['active_id'], context=context)
            if 'order_id' in fields:
                order_id = line.order_requirement_id.sale_order_id
                res.update(order_id=order_id.id)
        #     if 'product_id' in fields:
        #         res.update({'product_id': move.product_id.id})
        #     if 'product_uom' in fields:
        #         res.update({'product_uom': move.product_uom.id})
        #     if 'qty' in fields:
        #         res.update({'qty': move.product_qty})
        #     if 'use_exist' in fields:
        #         res.update({'use_exist': (move.picking_id and move.picking_id.type=='out' and True) or False})
        #     if 'location_id' in fields:
        #         res.update({'location_id': move.location_id.id})
        return res

    def _get_order_line(self, cr, uid, context=None):
        context = context or self.pool['res.users'].context_get(cr, uid)
        sale_order_line_obj = self.pool['sale.order.line']

        sale_order_line_ids = []
        if context.get('order_id'):
            line = self.pool['order.requirement'].browse(cr, uid, context['order_id'], context=context)
            order_id = line.sale_order_id.id
            sale_order_line_ids = sale_order_line_obj.search(cr, uid, [('order_id', '=', order_id)], context=context)
        res = sale_order_line_obj.name_get(cr, uid, sale_order_line_ids, context=context)

        return res

    _columns = {
        'order_id': fields.many2one('sale.order', 'Order', required=True, select=True),
        'order_line': fields.selection(_get_order_line, 'Order Line', required=False),
        # 'order_line_id': fields.many2one('sale.order.line', 'Production Lots'),
        # 'order_line_ids': fields.one2many('stock.move.split.lines', 'wizard_id', 'Production Lots'),
    }

    def link(self, cr, uid, ids, context=None):
        context = context or self.pool['res.users'].context_get(cr, uid)
        res = self._link(cr, uid, ids, context.get('active_ids'), context=context)
        return {'type': 'ir.actions.act_window_close'}

    def _link(self, cr, uid, ids, line_ids, context=None):
        context = context or self.pool['res.users'].context_get(cr, uid)
        for wizard in self.browse(cr, uid, ids, context):
            if wizard.order_line:
                line_id = int(wizard.order_line)
                order_requirement_line = self.pool['order.requirement.line'].browse(cr, uid, line_ids[0], context)
                order_requirement_line.write({'sale_order_line_id': line_id})

                order_requirement = self.pool['order.requirement'].browse(cr, uid, context['order_id'], context)

                # create picking
                pick_type = 'internal'
                ir_sequence_obj = self.pool['ir.sequence']
                stock_picking_obj = self.pool['stock.picking']

                pick_name = ir_sequence_obj.get(cr, uid, 'stock.picking.' + pick_type)
                order = order_requirement.sale_order_id
                date_planned = datetime.date.today().strftime(DEFAULT_SERVER_DATE_FORMAT)

                picking_vals = {
                    'name': pick_name,
                    'origin': _('Order Requirement') + ' ' + order.name,
                    'date': date_planned,
                    'type': pick_type,
                    'state': 'auto',
                    'move_type': 'one',
                    'sale_id': order.id,
                    'address_id': order.partner_shipping_id.id,
                    'note': order.note,
                    'invoice_state': 'none',
                    'company_id': order.company_id.id,
                    'auto_picking': True,
                }
                if order.project_project:
                    project = order.project_project
                    picking_vals.update({
                        'project_id': project.id,
                        'account_id': project.analytic_account_id.id,
                        'sale_project': project.id
                    })

                picking_id = stock_picking_obj.create(cr, uid, picking_vals, context)

                location_id = order.shop_id.warehouse_id.lot_stock_id.id
                output_id = order.shop_id.warehouse_id.lot_output_id.id
                price_unit = 0.0

                if order_requirement_line.qty != 0.0:
                    price_unit = order_requirement_line.product_id.cost_price

                move_vals = {
                    'name': order_requirement_line.product_id.name[:250],
                    'picking_id': picking_id,
                    'product_id': order_requirement_line.product_id.id,
                    'date': date_planned,
                    'date_expected': date_planned,
                    'product_qty': order_requirement_line.qty,
                    'product_uom': order_requirement_line.product_id.uom_id.id,
                    # 'product_uos_qty': (line.product_uos and line.product_uos_qty) or line.product_uom_qty,
                    # 'product_uos': (line.product_uos and line.product_uos.id) \
                    #                or line.product_uom.id,
                    # 'product_packaging': line.product_packaging.id,
                    # 'address_id': order.partner_shipping_id.id,
                    'location_id': location_id,
                    'location_dest_id': output_id,
                    'sale_line_id': line_id,
                    'tracking_id': False,
                    'state': 'draft',
                    # 'state': 'waiting',
                    'company_id': order.company_id.id,
                    'price_unit': price_unit
                }

                move_id = self.pool['stock.move'].create(cr, uid, move_vals, context)
                wf_service = netsvc.LocalService("workflow")
                wf_service.trg_validate(uid, 'stock.picking', picking_id, 'button_confirm', cr)
                stock_picking_obj.force_assign(cr, uid, [picking_id], context)
                ctx = context.copy()
                ctx['force_commit'] = True
                stock_picking_obj._commit_cost(cr, uid, [picking_id], ctx)

        return True
<|endoftext|>