<commit_before>import os,threading
import time

class TempMonitor(threading.Thread):
   def __init__(self,owner,sensor,quit_ev):
      threading.Thread.__init__(self)

      self.owner = owner # the KEgBot instance
      self.sensor = sensor # the TempSensor Quolz sensor
      self.QUIT = quit_ev # event to kill us

   def run(self):
      max_low = self.owner.config.getfloat('Thermo','temp_max_low')
      max_high = self.owner.config.getfloat('Thermo','temp_max_high')
      timeout = self.owner.config.getfloat('Thermo','logging_timeout')
      sensor = self.owner.config.getint('Thermo','main_sensor')
      lasttemp = -100.0
      last_time = 0
      while not self.QUIT.isSet():
         self.sensor.takeReading()
         currtemp,rectime = self.sensor.getTemp(sensor)

         if currtemp:
            if currtemp >= max_high:
               self.owner.enableFreezer()
            elif currtemp <= max_low:
               self.owner.disableFreezer()

            if time.time() - last_time > timeout:
               if currtemp != lasttemp:
                  fs = 0
                  if self.owner.fc.fridge_status:
                     fs = 1
                  self.owner.thermo_store.logTemp(currtemp, sensor, fs)
                  lasttemp = currtemp
               else:
                  self.owner.thermo_store.updateLastTemp()

               last_time = rectime

class TempSensor:
   """
   class for decoding output from a quozl temperature sensor.

   despite the confusing name, the tempsensor class does NOT represent a DS1820
   temperature sensor. it represents the quozl PIC device, which may have up to
   4 DS1820 sensors connected.
   """
   def __init__(self,dev,rate=2400):

      self.dev = dev
      self.rate = rate

      self._temps  = {}
      self._times  = {}
      self._version = "unknown"

      self._devpipe = open(dev,'w+',0) # unbuffered is zero

      try:
         os.system("stty %s raw < %s" % (self.rate, self.dev))
         pass
      except:
         print "error setting raw"
         pass

      # get rid of any data in the serial buffer; i'm only interested in the
      # most recent temperature
      self._devpipe.flush()

   def getTemp(self, sensor):
      """
      return the temperature of a particular sensor, as the tuple (temperature,
      time recorded)

      if a sensor has not been seen, or if the sensor number is invalid, a
      tuple of (None,0) is returned).
      """
      if not self._times.has_key(sensor):
         return (None,0)
      else:
         return (self._temps[sensor],self._times[sensor])

   def takeReading(self):
      """
      read one line (one temperature reading) from the quozl sensor.

      sensor readings are sent continuously to the serial port, seperated by
      \r\n and some spaces between fields. if a line is corrupt (or the
      decoding is out of alignment), no reading is taken.

      the first line the sensor sends on power-up is its version string. this
      qill be read and saved if encountered.
      """
      # gobble up the boundary characters
      while 1:
         c = self._devpipe.read(1)
         if c == '\r' or c == '\n':
            continue
         else:
            break

      # handle the power-on reset/info string
      if c == 'R':
         vers = ""
         c = self._devpipe.read(1)
         while c != '\r' and c != '\n':
            c = self._devpipe.read(1)
            vers += c
         self._version = vers

      # otherwise, grab the sensor data string
      else:
         # chomp space (or ignore corrupt/misaligned packet)
         space = self._devpipe.read(1)
         if space != ' ': return

         # we believe c contains a sensor number (0-4), but things could still
         # be misaligned. test.
         try:
            c = int(c)
         except:
            return

         amt = self._devpipe.read(7)
         try:
            self._temps[c] = float(amt)
            self._times[c] = time.time()
         except:
            return

<commit_msg>making fridge_status a function call, as it should be<commit_after>import os,threading
import time

class TempMonitor(threading.Thread):
   def __init__(self,owner,sensor,quit_ev):
      threading.Thread.__init__(self)

      self.owner = owner # the KEgBot instance
      self.sensor = sensor # the TempSensor Quolz sensor
      self.QUIT = quit_ev # event to kill us

   def run(self):
      max_low = self.owner.config.getfloat('Thermo','temp_max_low')
      max_high = self.owner.config.getfloat('Thermo','temp_max_high')
      timeout = self.owner.config.getfloat('Thermo','logging_timeout')
      sensor = self.owner.config.getint('Thermo','main_sensor')
      lasttemp = -100.0
      last_time = 0
      while not self.QUIT.isSet():
         self.sensor.takeReading()
         currtemp,rectime = self.sensor.getTemp(sensor)

         if currtemp:
            if currtemp >= max_high:
               self.owner.enableFreezer()
            elif currtemp <= max_low:
               self.owner.disableFreezer()

            if time.time() - last_time > timeout:
               if currtemp != lasttemp:
                  fs = 0
                  if self.owner.fc.fridgeStatus():
                     fs = 1
                  self.owner.thermo_store.logTemp(currtemp, sensor, fs)
                  lasttemp = currtemp
               else:
                  self.owner.thermo_store.updateLastTemp()

               last_time = rectime

class TempSensor:
   """
   class for decoding output from a quozl temperature sensor.

   despite the confusing name, the tempsensor class does NOT represent a DS1820
   temperature sensor. it represents the quozl PIC device, which may have up to
   4 DS1820 sensors connected.
   """
   def __init__(self,dev,rate=2400):

      self.dev = dev
      self.rate = rate

      self._temps  = {}
      self._times  = {}
      self._version = "unknown"

      self._devpipe = open(dev,'w+',0) # unbuffered is zero

      try:
         os.system("stty %s raw < %s" % (self.rate, self.dev))
         pass
      except:
         print "error setting raw"
         pass

      # get rid of any data in the serial buffer; i'm only interested in the
      # most recent temperature
      self._devpipe.flush()

   def getTemp(self, sensor):
      """
      return the temperature of a particular sensor, as the tuple (temperature,
      time recorded)

      if a sensor has not been seen, or if the sensor number is invalid, a
      tuple of (None,0) is returned).
      """
      if not self._times.has_key(sensor):
         return (None,0)
      else:
         return (self._temps[sensor],self._times[sensor])

   def takeReading(self):
      """
      read one line (one temperature reading) from the quozl sensor.

      sensor readings are sent continuously to the serial port, seperated by
      \r\n and some spaces between fields. if a line is corrupt (or the
      decoding is out of alignment), no reading is taken.

      the first line the sensor sends on power-up is its version string. this
      qill be read and saved if encountered.
      """
      # gobble up the boundary characters
      while 1:
         c = self._devpipe.read(1)
         if c == '\r' or c == '\n':
            continue
         else:
            break

      # handle the power-on reset/info string
      if c == 'R':
         vers = ""
         c = self._devpipe.read(1)
         while c != '\r' and c != '\n':
            c = self._devpipe.read(1)
            vers += c
         self._version = vers

      # otherwise, grab the sensor data string
      else:
         # chomp space (or ignore corrupt/misaligned packet)
         space = self._devpipe.read(1)
         if space != ' ': return

         # we believe c contains a sensor number (0-4), but things could still
         # be misaligned. test.
         try:
            c = int(c)
         except:
            return

         amt = self._devpipe.read(7)
         try:
            self._temps[c] = float(amt)
            self._times[c] = time.time()
         except:
            return

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

from resource_management import *
import os

def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or 'tracer' or 'client'
  import params

  # create the conf directory
  Directory( params.conf_dir,
      mode=0755,
      owner = params.accumulo_user,
      group = params.user_group,
      create_parents = True
  )

  if name == 'client':
    dest_conf_dir = params.conf_dir

    # create a site file for client processes
    configs = {}
    configs.update(params.config['configurations']['accumulo-site'])
    if "instance.secret" in configs:
      configs.pop("instance.secret")
    if "trace.token.property.password" in configs:
      configs.pop("trace.token.property.password")
    XmlConfig("accumulo-site.xml",
              conf_dir = dest_conf_dir,
              configurations = configs,
              configuration_attributes=params.config['configuration_attributes']['accumulo-site'],
              owner = params.accumulo_user,
              group = params.user_group,
              mode = 0644
    )

    # create env file
    File(format("{dest_conf_dir}/accumulo-env.sh"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(params.env_sh_template)
    )
  else:
    dest_conf_dir = params.server_conf_dir
    # create server conf directory
    Directory( params.server_conf_dir,
               mode=0700,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True
    )
    # create a site file for server processes
    configs = {}
    configs.update(params.config['configurations']['accumulo-site'])
    configs["instance.secret"] = str(params.config['configurations']['accumulo-env']['instance_secret'])
    configs["trace.token.property.password"] = str(params.trace_password)
    XmlConfig( "accumulo-site.xml",
               conf_dir = dest_conf_dir,
               configurations = configs,
               configuration_attributes=params.config['configuration_attributes']['accumulo-site'],
               owner = params.accumulo_user,
               group = params.user_group,
               mode = 0600
    )

    # create pid dir
    Directory( params.pid_dir,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True,
               cd_access = "a",
               mode = 0755,
    )

    # create log dir
    Directory (params.log_dir,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True,
               cd_access = "a",
               mode = 0755,
    )

    # create env file
    File(format("{dest_conf_dir}/accumulo-env.sh"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(params.server_env_sh_template)
    )

  # create client.conf file
  configs = {}
  if 'client' in params.config['configurations']:
    configs.update(params.config['configurations']['client'])
  configs["instance.name"] = params.instance_name
  configs["instance.zookeeper.host"] = params.config['configurations']['accumulo-site']['instance.zookeeper.host']
  copy_site_property(configs, 'instance.rpc.sasl.enabled')
  copy_site_property(configs, 'rpc.sasl.qop')
  copy_site_property(configs, 'rpc.useJsse')
  copy_site_property(configs, 'instance.rpc.ssl.clientAuth')
  copy_site_property(configs, 'instance.rpc.ssl.enabled')
  copy_site_property(configs, 'instance.zookeeper.timeout')
  copy_site_property(configs, 'trace.span.receivers')
  copy_site_property(configs, 'trace.zookeeper.path')
  for key,value in params.config['configurations']['accumulo-site'].iteritems():
    if key.startswith("trace.span.receiver."):
      configs[key] = value
  PropertiesFile(format("{dest_conf_dir}/client.conf"),
                 properties = configs,
                 owner = params.accumulo_user,
                 group = params.user_group
  )

  # create log4j.properties files
  if (params.log4j_props != None):
    File(format("{dest_conf_dir}/log4j.properties"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=params.log4j_props
    )
  else:
    File(format("{dest_conf_dir}/log4j.properties"),
         mode=0644,
         group=params.user_group,
         owner=params.hbase_user
    )

  # create logging configuration files
  accumulo_TemplateConfig("auditLog.xml", dest_conf_dir)
  accumulo_TemplateConfig("generic_logger.xml", dest_conf_dir)
  accumulo_TemplateConfig("monitor_logger.xml", dest_conf_dir)
  accumulo_StaticFile("accumulo-metrics.xml", dest_conf_dir)

  # create host files
  accumulo_TemplateConfig("tracers", dest_conf_dir)
  accumulo_TemplateConfig("gc", dest_conf_dir)
  accumulo_TemplateConfig("monitor", dest_conf_dir)
  accumulo_TemplateConfig("slaves", dest_conf_dir)
  accumulo_TemplateConfig("masters", dest_conf_dir)

  # metrics configuration
  if params.has_metric_collector:
    accumulo_TemplateConfig( "hadoop-metrics2-accumulo.properties", dest_conf_dir)

  # other server setup
  if name == 'master':
    params.HdfsResource(format("/user/{params.accumulo_user}"),
                         type="directory",
                         action="create_on_execute",
                         owner=params.accumulo_user,
                         mode=0700
    )
    params.HdfsResource(format("{params.parent_dir}"),
                         type="directory",
                         action="create_on_execute",
                         owner=params.accumulo_user,
                         mode=0700
    )
    params.HdfsResource(None, action="execute")
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} "
                      "{params.daemon_script} init "
                      "--user {params.accumulo_principal_name} "
                      "--instance-name {params.instance_name} "
                      "--clear-instance-name "
                      ">{params.log_dir}/accumulo-init.out "
                      "2>{params.log_dir}/accumulo-init.err"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.hadoop_bin_dir}/hadoop --config "
                                     "{params.hadoop_conf_dir} fs -stat "
                                     "{params.instance_volumes}"),
                              params.accumulo_user),
               logoutput=True,
               user=params.accumulo_user)
    else:
      passfile = format("{params.exec_tmp_dir}/pass")
      try:
        File(passfile,
             mode=0600,
             group=params.user_group,
             owner=params.accumulo_user,
             content=InlineTemplate('{{root_password}}\n'
                                    '{{root_password}}\n\n')
        )
        Execute( format("cat {passfile} | {params.daemon_script} init "
                        "--instance-name {params.instance_name} "
                        "--clear-instance-name "
                        ">{params.log_dir}/accumulo-init.out "
                        "2>{params.log_dir}/accumulo-init.err"),
                 not_if=as_user(format("{params.kinit_cmd} "
                                       "{params.hadoop_bin_dir}/hadoop --config "
                                       "{params.hadoop_conf_dir} fs -stat "
                                       "{params.instance_volumes}"),
                                params.accumulo_user),
                 logoutput=True,
                 user=params.accumulo_user)
      finally:
        os.remove(passfile)

  if name == 'tracer':
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} "
                      "{params.daemon_script} init --reset-security "
                      "--user {params.accumulo_principal_name} "
                      "--password NA "
                      ">{params.log_dir}/accumulo-reset.out "
                      "2>{params.log_dir}/accumulo-reset.err"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.daemon_script} shell -e "
                                     "\"userpermissions -u "
                                     "{params.accumulo_principal_name}\" | "
                                     "grep System.CREATE_TABLE"),
                              params.accumulo_user),
               user=params.accumulo_user)
      create_user(params.smokeuser_principal, params.smoke_test_password)
    else:
      # do not try to reset security in nonsecure mode, for now
      # Execute( format("{params.daemon_script} init --reset-security "
      #                 "--user root "
      #                 ">{params.log_dir}/accumulo-reset.out "
      #                 "2>{params.log_dir}/accumulo-reset.err"),
      #          not_if=as_user(format("cat {rpassfile} | "
      #                                "{params.daemon_script} shell -e "
      #                                "\"userpermissions -u root\" | "
      #                                "grep System.CREATE_TABLE"),
      #                         params.accumulo_user),
      #          user=params.accumulo_user)
      create_user(params.smoke_test_user, params.smoke_test_password)
    create_user(params.trace_user, params.trace_password)
    rpassfile = format("{params.exec_tmp_dir}/pass0")
    cmdfile = format("{params.exec_tmp_dir}/resetcmds")
    try:
      File(cmdfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate('grant -t trace -u {{trace_user}} Table.ALTER_TABLE\n'
                                  'grant -t trace -u {{trace_user}} Table.READ\n'
                                  'grant -t trace -u {{trace_user}} Table.WRITE\n\n')
      )
      if params.security_enabled and params.has_secure_user_auth:
        Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
                        "{cmdfile}"),
                 only_if=as_user(format("{params.kinit_cmd} "
                                        "{params.daemon_script} shell "
                                        "-e \"table trace\""),
                                 params.accumulo_user),
                 not_if=as_user(format("{params.kinit_cmd} "
                                       "{params.daemon_script} shell "
                                       "-e \"userpermissions -u "
                                       "{params.trace_user} | "
                                       "grep Table.READ | grep trace"),
                                params.accumulo_user),
                 user=params.accumulo_user)
      else:
        File(rpassfile,
             mode=0600,
             group=params.user_group,
             owner=params.accumulo_user,
             content=InlineTemplate('{{root_password}}\n\n')
        )
        Execute( format("cat {rpassfile} | {params.daemon_script} shell -f "
                        "{cmdfile} -u root"),
                 only_if=as_user(format("cat {rpassfile} | "
                                       "{params.daemon_script} shell -u root "
                                       "-e \"table trace\""),
                                params.accumulo_user),
                 not_if=as_user(format("cat {rpassfile} | "
                                       "{params.daemon_script} shell -u root "
                                       "-e \"userpermissions -u "
                                       "{params.trace_user} | "
                                       "grep Table.READ | grep trace"),
                                params.accumulo_user),
                 user=params.accumulo_user)
    finally:
      try_remove(rpassfile)
      try_remove(cmdfile)

def copy_site_property(configs, name):
  import params
  if name in params.config['configurations']['accumulo-site']:
    configs[name] = params.config['configurations']['accumulo-site'][name]

def create_user(user, password):
  import params
  rpassfile = format("{params.exec_tmp_dir}/pass0")
  passfile = format("{params.exec_tmp_dir}/pass")
  cmdfile = format("{params.exec_tmp_dir}/cmds")
  try:
    File(cmdfile,
         mode=0600,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(format("createuser {user}\n"
                                       "grant -s System.CREATE_TABLE -u {user}\n\n"))
    )
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
                      "{cmdfile}"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.daemon_script} shell "
                                     "-e \"userpermissions -u {user}\""),
                              params.accumulo_user),
               user=params.accumulo_user)
    else:
      File(rpassfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate('{{root_password}}\n\n')
      )
      File(passfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate(format("{params.root_password}\n"
                                         "{password}\n"
                                         "{password}\n\n"))
      )
      Execute( format("cat {passfile} | {params.daemon_script} shell -u root "
                      "-f {cmdfile}"),
               not_if=as_user(format("cat {rpassfile} | "
                                     "{params.daemon_script} shell -u root "
                                     "-e \"userpermissions -u {user}\""),
                              params.accumulo_user),
               user=params.accumulo_user)
  finally:
    try_remove(rpassfile)
    try_remove(passfile)
    try_remove(cmdfile)

def try_remove(file):
  try:
    os.remove(file)
  except:
    pass

# create file 'name' from template
def accumulo_TemplateConfig(name, dest_conf_dir, tag=None):
  import params

  TemplateConfig( format("{dest_conf_dir}/{name}"),
      owner = params.accumulo_user,
      group = params.user_group,
      template_tag = tag
  )

# create static file 'name'
def accumulo_StaticFile(name, dest_conf_dir):
  import params

  File(format("{dest_conf_dir}/{name}"),
    mode=0644,
    group=params.user_group,
    owner=params.accumulo_user,
    content=StaticFile(name)
  )
<commit_msg>AMBARI-17123. Accumulo master start fails on non-root + umask 027 (aonishuk)<commit_after>#!/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

from resource_management import *
import os

def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or 'tracer' or 'client'
  import params

  # create the conf directory
  Directory( params.conf_dir,
      mode=0755,
      owner = params.accumulo_user,
      group = params.user_group,
      create_parents = True
  )

  if name == 'client':
    dest_conf_dir = params.conf_dir

    # create a site file for client processes
    configs = {}
    configs.update(params.config['configurations']['accumulo-site'])
    if "instance.secret" in configs:
      configs.pop("instance.secret")
    if "trace.token.property.password" in configs:
      configs.pop("trace.token.property.password")
    XmlConfig("accumulo-site.xml",
              conf_dir = dest_conf_dir,
              configurations = configs,
              configuration_attributes=params.config['configuration_attributes']['accumulo-site'],
              owner = params.accumulo_user,
              group = params.user_group,
              mode = 0644
    )

    # create env file
    File(format("{dest_conf_dir}/accumulo-env.sh"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(params.env_sh_template)
    )
  else:
    dest_conf_dir = params.server_conf_dir
    # create server conf directory
    Directory( params.server_conf_dir,
               mode=0700,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True
    )
    # create a site file for server processes
    configs = {}
    configs.update(params.config['configurations']['accumulo-site'])
    configs["instance.secret"] = str(params.config['configurations']['accumulo-env']['instance_secret'])
    configs["trace.token.property.password"] = str(params.trace_password)
    XmlConfig( "accumulo-site.xml",
               conf_dir = dest_conf_dir,
               configurations = configs,
               configuration_attributes=params.config['configuration_attributes']['accumulo-site'],
               owner = params.accumulo_user,
               group = params.user_group,
               mode = 0600
    )

    # create pid dir
    Directory( params.pid_dir,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True,
               cd_access = "a",
               mode = 0755,
    )

    # create log dir
    Directory (params.log_dir,
               owner = params.accumulo_user,
               group = params.user_group,
               create_parents = True,
               cd_access = "a",
               mode = 0755,
    )

    # create env file
    File(format("{dest_conf_dir}/accumulo-env.sh"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(params.server_env_sh_template)
    )

  # create client.conf file
  configs = {}
  if 'client' in params.config['configurations']:
    configs.update(params.config['configurations']['client'])
  configs["instance.name"] = params.instance_name
  configs["instance.zookeeper.host"] = params.config['configurations']['accumulo-site']['instance.zookeeper.host']
  copy_site_property(configs, 'instance.rpc.sasl.enabled')
  copy_site_property(configs, 'rpc.sasl.qop')
  copy_site_property(configs, 'rpc.useJsse')
  copy_site_property(configs, 'instance.rpc.ssl.clientAuth')
  copy_site_property(configs, 'instance.rpc.ssl.enabled')
  copy_site_property(configs, 'instance.zookeeper.timeout')
  copy_site_property(configs, 'trace.span.receivers')
  copy_site_property(configs, 'trace.zookeeper.path')
  for key,value in params.config['configurations']['accumulo-site'].iteritems():
    if key.startswith("trace.span.receiver."):
      configs[key] = value
  PropertiesFile(format("{dest_conf_dir}/client.conf"),
                 properties = configs,
                 owner = params.accumulo_user,
                 group = params.user_group
  )

  # create log4j.properties files
  if (params.log4j_props != None):
    File(format("{dest_conf_dir}/log4j.properties"),
         mode=0644,
         group=params.user_group,
         owner=params.accumulo_user,
         content=params.log4j_props
    )
  else:
    File(format("{dest_conf_dir}/log4j.properties"),
         mode=0644,
         group=params.user_group,
         owner=params.hbase_user
    )

  # create logging configuration files
  accumulo_TemplateConfig("auditLog.xml", dest_conf_dir)
  accumulo_TemplateConfig("generic_logger.xml", dest_conf_dir)
  accumulo_TemplateConfig("monitor_logger.xml", dest_conf_dir)
  accumulo_StaticFile("accumulo-metrics.xml", dest_conf_dir)

  # create host files
  accumulo_TemplateConfig("tracers", dest_conf_dir)
  accumulo_TemplateConfig("gc", dest_conf_dir)
  accumulo_TemplateConfig("monitor", dest_conf_dir)
  accumulo_TemplateConfig("slaves", dest_conf_dir)
  accumulo_TemplateConfig("masters", dest_conf_dir)

  # metrics configuration
  if params.has_metric_collector:
    accumulo_TemplateConfig( "hadoop-metrics2-accumulo.properties", dest_conf_dir)

  # other server setup
  if name == 'master':
    params.HdfsResource(format("/user/{params.accumulo_user}"),
                         type="directory",
                         action="create_on_execute",
                         owner=params.accumulo_user,
                         mode=0700
    )
    params.HdfsResource(format("{params.parent_dir}"),
                         type="directory",
                         action="create_on_execute",
                         owner=params.accumulo_user,
                         mode=0700
    )
    params.HdfsResource(None, action="execute")
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} "
                      "{params.daemon_script} init "
                      "--user {params.accumulo_principal_name} "
                      "--instance-name {params.instance_name} "
                      "--clear-instance-name "
                      ">{params.log_dir}/accumulo-init.out "
                      "2>{params.log_dir}/accumulo-init.err"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.hadoop_bin_dir}/hadoop --config "
                                     "{params.hadoop_conf_dir} fs -stat "
                                     "{params.instance_volumes}"),
                              params.accumulo_user),
               logoutput=True,
               user=params.accumulo_user)
    else:
      passfile = format("{params.exec_tmp_dir}/pass")
      try:
        File(passfile,
             mode=0600,
             group=params.user_group,
             owner=params.accumulo_user,
             content=InlineTemplate('{{root_password}}\n'
                                    '{{root_password}}\n\n')
        )
        Execute( format("cat {passfile} | {params.daemon_script} init "
                        "--instance-name {params.instance_name} "
                        "--clear-instance-name "
                        ">{params.log_dir}/accumulo-init.out "
                        "2>{params.log_dir}/accumulo-init.err"),
                 not_if=as_user(format("{params.kinit_cmd} "
                                       "{params.hadoop_bin_dir}/hadoop --config "
                                       "{params.hadoop_conf_dir} fs -stat "
                                       "{params.instance_volumes}"),
                                params.accumulo_user),
                 logoutput=True,
                 user=params.accumulo_user)
      finally:
        File(passfile, action = "delete")

  if name == 'tracer':
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} "
                      "{params.daemon_script} init --reset-security "
                      "--user {params.accumulo_principal_name} "
                      "--password NA "
                      ">{params.log_dir}/accumulo-reset.out "
                      "2>{params.log_dir}/accumulo-reset.err"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.daemon_script} shell -e "
                                     "\"userpermissions -u "
                                     "{params.accumulo_principal_name}\" | "
                                     "grep System.CREATE_TABLE"),
                              params.accumulo_user),
               user=params.accumulo_user)
      create_user(params.smokeuser_principal, params.smoke_test_password)
    else:
      # do not try to reset security in nonsecure mode, for now
      # Execute( format("{params.daemon_script} init --reset-security "
      #                 "--user root "
      #                 ">{params.log_dir}/accumulo-reset.out "
      #                 "2>{params.log_dir}/accumulo-reset.err"),
      #          not_if=as_user(format("cat {rpassfile} | "
      #                                "{params.daemon_script} shell -e "
      #                                "\"userpermissions -u root\" | "
      #                                "grep System.CREATE_TABLE"),
      #                         params.accumulo_user),
      #          user=params.accumulo_user)
      create_user(params.smoke_test_user, params.smoke_test_password)
    create_user(params.trace_user, params.trace_password)
    rpassfile = format("{params.exec_tmp_dir}/pass0")
    cmdfile = format("{params.exec_tmp_dir}/resetcmds")
    try:
      File(cmdfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate('grant -t trace -u {{trace_user}} Table.ALTER_TABLE\n'
                                  'grant -t trace -u {{trace_user}} Table.READ\n'
                                  'grant -t trace -u {{trace_user}} Table.WRITE\n\n')
      )
      if params.security_enabled and params.has_secure_user_auth:
        Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
                        "{cmdfile}"),
                 only_if=as_user(format("{params.kinit_cmd} "
                                        "{params.daemon_script} shell "
                                        "-e \"table trace\""),
                                 params.accumulo_user),
                 not_if=as_user(format("{params.kinit_cmd} "
                                       "{params.daemon_script} shell "
                                       "-e \"userpermissions -u "
                                       "{params.trace_user} | "
                                       "grep Table.READ | grep trace"),
                                params.accumulo_user),
                 user=params.accumulo_user)
      else:
        File(rpassfile,
             mode=0600,
             group=params.user_group,
             owner=params.accumulo_user,
             content=InlineTemplate('{{root_password}}\n\n')
        )
        Execute( format("cat {rpassfile} | {params.daemon_script} shell -f "
                        "{cmdfile} -u root"),
                 only_if=as_user(format("cat {rpassfile} | "
                                       "{params.daemon_script} shell -u root "
                                       "-e \"table trace\""),
                                params.accumulo_user),
                 not_if=as_user(format("cat {rpassfile} | "
                                       "{params.daemon_script} shell -u root "
                                       "-e \"userpermissions -u "
                                       "{params.trace_user} | "
                                       "grep Table.READ | grep trace"),
                                params.accumulo_user),
                 user=params.accumulo_user)
    finally:
      try_remove(rpassfile)
      try_remove(cmdfile)

def copy_site_property(configs, name):
  import params
  if name in params.config['configurations']['accumulo-site']:
    configs[name] = params.config['configurations']['accumulo-site'][name]

def create_user(user, password):
  import params
  rpassfile = format("{params.exec_tmp_dir}/pass0")
  passfile = format("{params.exec_tmp_dir}/pass")
  cmdfile = format("{params.exec_tmp_dir}/cmds")
  try:
    File(cmdfile,
         mode=0600,
         group=params.user_group,
         owner=params.accumulo_user,
         content=InlineTemplate(format("createuser {user}\n"
                                       "grant -s System.CREATE_TABLE -u {user}\n\n"))
    )
    if params.security_enabled and params.has_secure_user_auth:
      Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
                      "{cmdfile}"),
               not_if=as_user(format("{params.kinit_cmd} "
                                     "{params.daemon_script} shell "
                                     "-e \"userpermissions -u {user}\""),
                              params.accumulo_user),
               user=params.accumulo_user)
    else:
      File(rpassfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate('{{root_password}}\n\n')
      )
      File(passfile,
           mode=0600,
           group=params.user_group,
           owner=params.accumulo_user,
           content=InlineTemplate(format("{params.root_password}\n"
                                         "{password}\n"
                                         "{password}\n\n"))
      )
      Execute( format("cat {passfile} | {params.daemon_script} shell -u root "
                      "-f {cmdfile}"),
               not_if=as_user(format("cat {rpassfile} | "
                                     "{params.daemon_script} shell -u root "
                                     "-e \"userpermissions -u {user}\""),
                              params.accumulo_user),
               user=params.accumulo_user)
  finally:
    try_remove(rpassfile)
    try_remove(passfile)
    try_remove(cmdfile)

def try_remove(file):
  try:
    os.remove(file)
  except:
    pass

# create file 'name' from template
def accumulo_TemplateConfig(name, dest_conf_dir, tag=None):
  import params

  TemplateConfig( format("{dest_conf_dir}/{name}"),
      owner = params.accumulo_user,
      group = params.user_group,
      template_tag = tag
  )

# create static file 'name'
def accumulo_StaticFile(name, dest_conf_dir):
  import params

  File(format("{dest_conf_dir}/{name}"),
    mode=0644,
    group=params.user_group,
    owner=params.accumulo_user,
    content=StaticFile(name)
  )
<|endoftext|>


<commit_before>from __future__ import print_function
from flask import render_template, redirect, url_for, abort, flash, request,\
    current_app, make_response
from flask_login import login_required, current_user
from flask_sqlalchemy import get_debug_queries
from . import main
from .forms import EditProfileForm, EditProfileAdminForm, PostForm,\
    CommentForm
from .. import db
from ..models import Permission, Role, User, Post, Comment, Workflow
from ..decorators import admin_required, permission_required
import os
import sys
from hdfs import InsecureClient

@main.after_app_request
def after_request(response):
    for query in get_debug_queries():
        if query.duration >= current_app.config['PHENOPROC_SLOW_DB_QUERY_TIME']:
            current_app.logger.warning(
                'Slow query: %s\nParameters: %s\nDuration: %fs\nContext: %s\n'
                % (query.statement, query.parameters, query.duration,
                   query.context))
    return response

@main.route('/shutdown')
def server_shutdown():
    if not current_app.testing:
        abort(404)
    shutdown = request.environ.get('werkzeug.server.shutdown')
    if not shutdown:
        abort(500)
    shutdown()
    return 'Shutting down...'

def make_fs_tree(path):
    tree = dict(name=os.path.basename(path), children=[])
    try: lst = os.listdir(path)
    except OSError:
        pass #ignore errors
    else:
        for name in lst:
            fn = os.path.join(path, name)
            if os.path.isdir(fn):
                tree['children'].append(make_fs_tree(fn))
            else:
                tree['children'].append(dict(name=name))
    return tree

def make_hdfs_tree(client, path):
    tree = dict(name=os.path.basename(path), children=[])
    try: lst = client.list(path, status=True)
    except:
        pass #ignore errors
    else:
        for name in lst:
            fn = os.path.join(path, name[0])
            if name[1]['type'] == "DIRECTORY":
                tree['children'].append(make_hdfs_tree(fn))
            else:
                tree['children'].append(dict(name=name[0]))
    return tree

@main.route('/', methods=['GET', 'POST'])
def index():
    form = PostForm()
    if current_user.can(Permission.WRITE_ARTICLES) and \
            form.validate_on_submit():
        post = Post(body=form.body.data,
                    author=current_user._get_current_object())
        db.session.add(post)
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    show_followed = False
    if current_user.is_authenticated:
        show_followed = bool(request.cookies.get('show_followed', ''))
    if show_followed:
        query = current_user.followed_posts
    else:
        query = Post.query
    pagination = query.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items

    datasources = { 'name' : 'datasources', 'children' : [] }
    datasources['children'].append({ 'name' : 'Phenodoop', 'children' : [] })
    datasources['children'].append({ 'name' : 'File System', 'children' : [] })

#    print(datasources['children'].len, file=sys.stderr)
    try:
        client = InsecureClient(current_app.config['WEBHDFS_ADDR'], user=current_app.config['WEBHDFS_USER'])
    except:
        pass
    else:
        hdfs_tree = datasources['children'][0]['children']
        if client is not None:
            if current_user.is_authenticated:
                hdfs_tree.append(make_hdfs_tree(os.path.join(current_app.config['HDFS_DIR'], current_user.username)))
            hdfs_tree.append(make_hdfs_tree(os.path.join(current_app.config['HDFS_DIR'], 'public')))

    fs_tree = datasources['children'][1]['children']
    if current_user.is_authenticated and os.path.exists(os.path.join(current_app.config['DATA_DIR'], current_user.username)):
            fs_tree.append(make_fs_tree(os.path.join(current_app.config['DATA_DIR'], current_user.username)))

    fs_tree.append(make_fs_tree(os.path.join(current_app.config['DATA_DIR'], 'public')))

    return render_template('index.html', form=form, posts=posts, datasources=datasources,
                           show_followed=show_followed, pagination=pagination)

@main.route('/user/<username>')
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    page = request.args.get('page', 1, type=int)
    pagination = user.posts.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items
    return render_template('user.html', user=user, posts=posts,
                           pagination=pagination)

@main.route('/edit-profile', methods=['GET', 'POST'])
@login_required
def edit_profile():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.name = form.name.data
        current_user.location = form.location.data
        current_user.about_me = form.about_me.data
        db.session.add(current_user)
        flash('Your profile has been updated.')
        return redirect(url_for('.user', username=current_user.username))
    form.name.data = current_user.name
    form.location.data = current_user.location
    form.about_me.data = current_user.about_me
    return render_template('edit_profile.html', form=form)

@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])
@login_required
@admin_required
def edit_profile_admin(id):
    user = User.query.get_or_404(id)
    form = EditProfileAdminForm(user=user)
    if form.validate_on_submit():
        user.email = form.email.data
        user.username = form.username.data
        user.confirmed = form.confirmed.data
        user.role = Role.query.get(form.role.data)
        user.name = form.name.data
        user.location = form.location.data
        user.about_me = form.about_me.data
        db.session.add(user)
        flash('The profile has been updated.')
        return redirect(url_for('.user', username=user.username))
    form.email.data = user.email
    form.username.data = user.username
    form.confirmed.data = user.confirmed
    form.role.data = user.role_id
    form.name.data = user.name
    form.location.data = user.location
    form.about_me.data = user.about_me
    return render_template('edit_profile.html', form=form, user=user)

@main.route('/post/<int:id>', methods=['GET', 'POST'])
def post(id):
    post = Post.query.get_or_404(id)
    form = CommentForm()
    if form.validate_on_submit():
        comment = Comment(body=form.body.data,
                          post=post,
                          author=current_user._get_current_object())
        db.session.add(comment)
        flash('Your comment has been published.')
        return redirect(url_for('.post', id=post.id, page=-1))
    page = request.args.get('page', 1, type=int)
    if page == -1:
        page = (post.comments.count() - 1) // \
            current_app.config['PHENOPROC_COMMENTS_PER_PAGE'] + 1
    pagination = post.comments.order_by(Comment.timestamp.asc()).paginate(
        page, per_page=current_app.config['PHENOPROC_COMMENTS_PER_PAGE'],
        error_out=False)
    comments = pagination.items
    return render_template('post.html', posts=[post], form=form,
                           comments=comments, pagination=pagination)

@main.route('/workflow/<int:id>', methods=['GET', 'POST'])
def workflow(id):
    workflow = Workflow.query.get_or_404(id)
    return render_template('workflow.html', workflows=[workflow])

@main.route('/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit(id):
    post = Post.query.get_or_404(id)
    if current_user != post.author and \
            not current_user.can(Permission.ADMINISTER):
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.body = form.body.data
        db.session.add(post)
        flash('The post has been updated.')
        return redirect(url_for('.post', id=post.id))
    form.body.data = post.body
    return render_template('edit_post.html', form=form)

@main.route('/follow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if current_user.is_following(user):
        flash('You are already following this user.')
        return redirect(url_for('.user', username=username))
    current_user.follow(user)
    flash('You are now following %s.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/unfollow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if not current_user.is_following(user):
        flash('You are not following this user.')
        return redirect(url_for('.user', username=username))
    current_user.unfollow(user)
    flash('You are not following %s anymore.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/followers/<username>')
def followers(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followers.paginate(
        page, per_page=current_app.config['PHENOPROC_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.follower, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followers of",
                           endpoint='.followers', pagination=pagination,
                           follows=follows)

@main.route('/followed-by/<username>')
def followed_by(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followed.paginate(
        page, per_page=current_app.config['PHENOPROC_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.followed, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followed by",
                           endpoint='.followed_by', pagination=pagination,
                           follows=follows)

@main.route('/all')
@login_required
def show_all():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '', max_age=30*24*60*60)
    return resp

@main.route('/followed')
@login_required
def show_followed():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '1', max_age=30*24*60*60)
    return resp

@main.route('/moderate')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate():
    page = request.args.get('page', 1, type=int)
    pagination = Comment.query.order_by(Comment.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_COMMENTS_PER_PAGE'],
        error_out=False)
    comments = pagination.items
    return render_template('moderate.html', comments=comments,
                           pagination=pagination, page=page)

@main.route('/moderate/enable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate_enable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = False
    db.session.add(comment)
    return redirect(url_for('.moderate',
                            page=request.args.get('page', 1, type=int)))

@main.route('/moderate/disable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate_disable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = True
    db.session.add(comment)
    return redirect(url_for('.moderate',
                            page=request.args.get('page', 1, type=int)))

@main.route('/about')
def about():
    return render_template('about.html')

@main.route('/contact')
def contact():
    return render_template('contact.html')<commit_msg>hdfscli client added<commit_after>from __future__ import print_function
from flask import render_template, redirect, url_for, abort, flash, request,\
    current_app, make_response
from flask_login import login_required, current_user
from flask_sqlalchemy import get_debug_queries
from . import main
from .forms import EditProfileForm, EditProfileAdminForm, PostForm,\
    CommentForm
from .. import db
from ..models import Permission, Role, User, Post, Comment, Workflow
from ..decorators import admin_required, permission_required
import os
import sys
try:
    from hdfs import InsecureClient
except:
    pass

@main.after_app_request
def after_request(response):
    for query in get_debug_queries():
        if query.duration >= current_app.config['PHENOPROC_SLOW_DB_QUERY_TIME']:
            current_app.logger.warning(
                'Slow query: %s\nParameters: %s\nDuration: %fs\nContext: %s\n'
                % (query.statement, query.parameters, query.duration,
                   query.context))
    return response

@main.route('/shutdown')
def server_shutdown():
    if not current_app.testing:
        abort(404)
    shutdown = request.environ.get('werkzeug.server.shutdown')
    if not shutdown:
        abort(500)
    shutdown()
    return 'Shutting down...'

def make_fs_tree(path):
    tree = dict(name=os.path.basename(path), children=[])
    try: lst = os.listdir(path)
    except OSError:
        pass #ignore errors
    else:
        for name in lst:
            fn = os.path.join(path, name)
            if os.path.isdir(fn):
                tree['children'].append(make_fs_tree(fn))
            else:
                tree['children'].append(dict(name=name))
    return tree

def make_hdfs_tree(client, path):
    tree = dict(name=os.path.basename(path), children=[])
    try: lst = client.list(path, status=True)
    except:
        pass #ignore errors
    else:
        for name in lst:
            fn = os.path.join(path, name[0])
            if name[1]['type'] == "DIRECTORY":
                tree['children'].append(make_hdfs_tree(fn))
            else:
                tree['children'].append(dict(name=name[0]))
    return tree

@main.route('/', methods=['GET', 'POST'])
def index():
    form = PostForm()
    if current_user.can(Permission.WRITE_ARTICLES) and \
            form.validate_on_submit():
        post = Post(body=form.body.data,
                    author=current_user._get_current_object())
        db.session.add(post)
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    show_followed = False
    if current_user.is_authenticated:
        show_followed = bool(request.cookies.get('show_followed', ''))
    if show_followed:
        query = current_user.followed_posts
    else:
        query = Post.query
    pagination = query.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items

    datasources = { 'name' : 'datasources', 'children' : [] }
    datasources['children'].append({ 'name' : 'Phenodoop', 'children' : [] })
    datasources['children'].append({ 'name' : 'File System', 'children' : [] })

#    print(datasources['children'].len, file=sys.stderr)
    try:
        client = InsecureClient(current_app.config['WEBHDFS_ADDR'], user=current_app.config['WEBHDFS_USER'])
    except:
        pass
    else:
        hdfs_tree = datasources['children'][0]['children']
        if client is not None:
            if current_user.is_authenticated:
                hdfs_tree.append(make_hdfs_tree(os.path.join(current_app.config['HDFS_DIR'], current_user.username)))
            hdfs_tree.append(make_hdfs_tree(os.path.join(current_app.config['HDFS_DIR'], 'public')))

    fs_tree = datasources['children'][1]['children']
    if current_user.is_authenticated and os.path.exists(os.path.join(current_app.config['DATA_DIR'], current_user.username)):
            fs_tree.append(make_fs_tree(os.path.join(current_app.config['DATA_DIR'], current_user.username)))

    fs_tree.append(make_fs_tree(os.path.join(current_app.config['DATA_DIR'], 'public')))

    return render_template('index.html', form=form, posts=posts, datasources=datasources,
                           show_followed=show_followed, pagination=pagination)

@main.route('/user/<username>')
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    page = request.args.get('page', 1, type=int)
    pagination = user.posts.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items
    return render_template('user.html', user=user, posts=posts,
                           pagination=pagination)

@main.route('/edit-profile', methods=['GET', 'POST'])
@login_required
def edit_profile():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.name = form.name.data
        current_user.location = form.location.data
        current_user.about_me = form.about_me.data
        db.session.add(current_user)
        flash('Your profile has been updated.')
        return redirect(url_for('.user', username=current_user.username))
    form.name.data = current_user.name
    form.location.data = current_user.location
    form.about_me.data = current_user.about_me
    return render_template('edit_profile.html', form=form)

@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])
@login_required
@admin_required
def edit_profile_admin(id):
    user = User.query.get_or_404(id)
    form = EditProfileAdminForm(user=user)
    if form.validate_on_submit():
        user.email = form.email.data
        user.username = form.username.data
        user.confirmed = form.confirmed.data
        user.role = Role.query.get(form.role.data)
        user.name = form.name.data
        user.location = form.location.data
        user.about_me = form.about_me.data
        db.session.add(user)
        flash('The profile has been updated.')
        return redirect(url_for('.user', username=user.username))
    form.email.data = user.email
    form.username.data = user.username
    form.confirmed.data = user.confirmed
    form.role.data = user.role_id
    form.name.data = user.name
    form.location.data = user.location
    form.about_me.data = user.about_me
    return render_template('edit_profile.html', form=form, user=user)

@main.route('/post/<int:id>', methods=['GET', 'POST'])
def post(id):
    post = Post.query.get_or_404(id)
    form = CommentForm()
    if form.validate_on_submit():
        comment = Comment(body=form.body.data,
                          post=post,
                          author=current_user._get_current_object())
        db.session.add(comment)
        flash('Your comment has been published.')
        return redirect(url_for('.post', id=post.id, page=-1))
    page = request.args.get('page', 1, type=int)
    if page == -1:
        page = (post.comments.count() - 1) // \
            current_app.config['PHENOPROC_COMMENTS_PER_PAGE'] + 1
    pagination = post.comments.order_by(Comment.timestamp.asc()).paginate(
        page, per_page=current_app.config['PHENOPROC_COMMENTS_PER_PAGE'],
        error_out=False)
    comments = pagination.items
    return render_template('post.html', posts=[post], form=form,
                           comments=comments, pagination=pagination)

@main.route('/workflow/<int:id>', methods=['GET', 'POST'])
def workflow(id):
    workflow = Workflow.query.get_or_404(id)
    return render_template('workflow.html', workflows=[workflow])

@main.route('/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit(id):
    post = Post.query.get_or_404(id)
    if current_user != post.author and \
            not current_user.can(Permission.ADMINISTER):
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.body = form.body.data
        db.session.add(post)
        flash('The post has been updated.')
        return redirect(url_for('.post', id=post.id))
    form.body.data = post.body
    return render_template('edit_post.html', form=form)

@main.route('/follow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if current_user.is_following(user):
        flash('You are already following this user.')
        return redirect(url_for('.user', username=username))
    current_user.follow(user)
    flash('You are now following %s.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/unfollow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if not current_user.is_following(user):
        flash('You are not following this user.')
        return redirect(url_for('.user', username=username))
    current_user.unfollow(user)
    flash('You are not following %s anymore.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/followers/<username>')
def followers(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followers.paginate(
        page, per_page=current_app.config['PHENOPROC_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.follower, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followers of",
                           endpoint='.followers', pagination=pagination,
                           follows=follows)

@main.route('/followed-by/<username>')
def followed_by(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followed.paginate(
        page, per_page=current_app.config['PHENOPROC_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.followed, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followed by",
                           endpoint='.followed_by', pagination=pagination,
                           follows=follows)

@main.route('/all')
@login_required
def show_all():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '', max_age=30*24*60*60)
    return resp

@main.route('/followed')
@login_required
def show_followed():
    resp = make_response(redirect(url_for('.index')))
    resp.set_cookie('show_followed', '1', max_age=30*24*60*60)
    return resp

@main.route('/moderate')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate():
    page = request.args.get('page', 1, type=int)
    pagination = Comment.query.order_by(Comment.timestamp.desc()).paginate(
        page, per_page=current_app.config['PHENOPROC_COMMENTS_PER_PAGE'],
        error_out=False)
    comments = pagination.items
    return render_template('moderate.html', comments=comments,
                           pagination=pagination, page=page)

@main.route('/moderate/enable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate_enable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = False
    db.session.add(comment)
    return redirect(url_for('.moderate',
                            page=request.args.get('page', 1, type=int)))

@main.route('/moderate/disable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def moderate_disable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = True
    db.session.add(comment)
    return redirect(url_for('.moderate',
                            page=request.args.get('page', 1, type=int)))

@main.route('/about')
def about():
    return render_template('about.html')

@main.route('/contact')
def contact():
    return render_template('contact.html')<|endoftext|>


<commit_before><commit_msg>Added better logging of crashes.<commit_after><|endoftext|>


<commit_before># Need something to make sense of a bunch of downloaded images....
# Vid-000512014-00-03-2009-12-17-17-42.jpg
#           Site Identifier
#               CameraID (may have more than one camera in the future)
#                  ViewID
#                     GMT Timestamp....

FTP_PASS = "----"

import os, mx.DateTime, glob, re
from PIL import Image, ImageDraw, ImageFont
font = ImageFont.truetype('veramono.ttf', 10)
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
mesosite = i['mesosite']
gmt = mx.DateTime.gmt()

# Make dictionary of webcams we are interested in
cameras = {}
rs = mesosite.query("SELECT * from webcams WHERE network = 'IDOT'").dictresult()
for i in range(len(rs)):
  cameras[ rs[i]['id'] ] = rs[i]

os.chdir("work")
si, so = os.popen4("wget  -m --ftp-user=rwis --ftp-password=%s \
           ftp://165.206.203.34/rwis_images/*%s-??.jpg" % (FTP_PASS,
           gmt.strftime("%H")))

data = so.read()
lines = data.split("\n")
for line in lines:
  # Look for RETR (.*)
  tokens = re.findall("RETR Vid-000512([0-9]{3})-([0-9][0-9])-([0-9][0-9])-([0-9]{4})-([0-9][0-9])-([0-9][0-9])-([0-9][0-9])-([0-9][0-9]).jpg", line)
  if len(tokens) == 0:
    continue

  t = tokens[0]
  cid = "IDOT-%s-%s" % (t[0], t[2])
  gmt = mx.DateTime.DateTime( int(t[3]), int(t[4]), int(t[5]),
                             int(t[6]), int(t[7]) )
  now = gmt.localtime()
  if not cameras.has_key(cid):
    print "Unknown CameraID: %s" % (cid,)
    cameras[ cid ] = {'pan0': 0, 'name': 'unknown'}

  # Hard coded...
  d = cameras[ cid ]
  drct = d['pan0']
  drctTxt = mesonet.drct2dirTxt( drct )

  # Create 320x240 variant
  fp = "165.206.203.34/rwis_images/Vid-000512%s.jpg" % ("-".join(t),)
  try:
    i0 = Image.open( fp )
    i320 = i0.resize((320, 240), Image.ANTIALIAS)
  except:
    print "ERROR: ", fp
    os.unlink( fp )
    continue

  draw = ImageDraw.Draw(i0)
  str = "(%s) %s %s" % (drctTxt, d['name'], now.strftime("%-2I:%M:%S %p - %d %b %Y") )
  (w, h) = font.getsize(str)
  draw.rectangle( [5,475-h,5+w,475], fill="#000000" )
  draw.text((5,475-h), str, font=font)

  # Save 640x480
  i0.save("%s-640x480.jpg" % (cid,) )

  draw = ImageDraw.Draw(i320)
  str = "(%s) %s %s" % (drctTxt, d['name'], now.strftime("%-2I:%M:%S %p - %d %b %Y") )
  (w, h) = font.getsize(str)
  draw.rectangle( [5,235-h,5+w,235], fill="#000000" )
  draw.text((5,235-h), str, font=font)

  # Save 640x480
  i320.save("%s-320x240.jpg" % (cid,) )
  del(i0)
  del(i320)

  # Insert into LDM
  cmd = "/home/ldm/bin/pqinsert -p 'webcam ac %s camera/stills/%s.jpg camera/%s/%s_%s.jpg jpg' %s-320x240.jpg" % (gmt.strftime("%Y%m%d%H%M"), cid, cid, cid, gmt.strftime("%Y%m%d%H%M"), cid)
  os.system(cmd)
  cmd = "/home/ldm/bin/pqinsert -p 'webcam c %s camera/640x480/%s.jpg bogus jpg' %s-640x480.jpg" % (gmt.strftime("%Y%m%d%H%M"), cid, cid )
  os.system(cmd)

  # Insert into webcam log please
  sql = """INSERT into camera_log (cam, valid, drct) VALUES
           ('%s', '%s', %s)""" % (cid, now.strftime("%Y-%m-%d %H:%I"),
           drct)
  mesosite.query( sql )

  sql = "DELETE from camera_current WHERE cam = '%s'" % (cid,)
  mesosite.query(sql)

  sql = "INSERT into camera_current(cam, valid, drct) values ('%s','%s',%s)" \
   %(cid, now.strftime('%Y-%m-%d %H:%M'), drct)
  mesosite.query(sql)

<commit_msg>Cleanup mess and don't be so noisey<commit_after># Need something to make sense of a bunch of downloaded images....
# Vid-000512014-00-03-2009-12-17-17-42.jpg
#           Site Identifier
#               CameraID (may have more than one camera in the future)
#                  ViewID
#                     GMT Timestamp....

FTP_PASS = "----"

import os, mx.DateTime, glob, re
from PIL import Image, ImageDraw, ImageFont
font = ImageFont.truetype('veramono.ttf', 10)
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
mesosite = i['mesosite']
gmt = mx.DateTime.gmt()

# Every three hours, clean up after ourselves :)
if gmt.hour % 3 == 0 and gmt.minute < 5:
  os.system("/usr/sbin/tmpwatch 6 165.206.203.34/rwis_images")

# Make dictionary of webcams we are interested in
cameras = {}
rs = mesosite.query("SELECT * from webcams WHERE network = 'IDOT'").dictresult()
for i in range(len(rs)):
  cameras[ rs[i]['id'] ] = rs[i]

os.chdir("work")
si, so = os.popen4("wget  -m --ftp-user=rwis --ftp-password=%s \
           ftp://165.206.203.34/rwis_images/*%s-??.jpg" % (FTP_PASS,
           gmt.strftime("%H")))

data = so.read()
lines = data.split("\n")
for line in lines:
  # Look for RETR (.*)
  tokens = re.findall("RETR Vid-000512([0-9]{3})-([0-9][0-9])-([0-9][0-9])-([0-9]{4})-([0-9][0-9])-([0-9][0-9])-([0-9][0-9])-([0-9][0-9]).jpg", line)
  if len(tokens) == 0:
    continue

  t = tokens[0]
  cid = "IDOT-%s-%s" % (t[0], t[2])
  gmt = mx.DateTime.DateTime( int(t[3]), int(t[4]), int(t[5]),
                             int(t[6]), int(t[7]) )
  now = gmt.localtime()
  if not cameras.has_key(cid):
    print "Unknown CameraID: %s" % (cid,)
    cameras[ cid ] = {'pan0': 0, 'name': 'unknown'}

  # Hard coded...
  d = cameras[ cid ]
  drct = d['pan0']
  drctTxt = mesonet.drct2dirTxt( drct )

  # Create 320x240 variant
  fp = "165.206.203.34/rwis_images/Vid-000512%s.jpg" % ("-".join(t),)
  try:
    i0 = Image.open( fp )
    i320 = i0.resize((320, 240), Image.ANTIALIAS)
  except:
    #print "ERROR: ", fp
    os.unlink( fp )
    continue

  draw = ImageDraw.Draw(i0)
  str = "(%s) %s %s" % (drctTxt, d['name'], now.strftime("%-2I:%M:%S %p - %d %b %Y") )
  (w, h) = font.getsize(str)
  draw.rectangle( [5,475-h,5+w,475], fill="#000000" )
  draw.text((5,475-h), str, font=font)

  # Save 640x480
  i0.save("%s-640x480.jpg" % (cid,) )

  draw = ImageDraw.Draw(i320)
  str = "(%s) %s %s" % (drctTxt, d['name'], now.strftime("%-2I:%M:%S %p - %d %b %Y") )
  (w, h) = font.getsize(str)
  draw.rectangle( [5,235-h,5+w,235], fill="#000000" )
  draw.text((5,235-h), str, font=font)

  # Save 640x480
  i320.save("%s-320x240.jpg" % (cid,) )
  del(i0)
  del(i320)

  # Insert into LDM
  cmd = "/home/ldm/bin/pqinsert -p 'webcam ac %s camera/stills/%s.jpg camera/%s/%s_%s.jpg jpg' %s-320x240.jpg" % (gmt.strftime("%Y%m%d%H%M"), cid, cid, cid, gmt.strftime("%Y%m%d%H%M"), cid)
  os.system(cmd)
  cmd = "/home/ldm/bin/pqinsert -p 'webcam c %s camera/640x480/%s.jpg bogus jpg' %s-640x480.jpg" % (gmt.strftime("%Y%m%d%H%M"), cid, cid )
  os.system(cmd)

  # Insert into webcam log please
  sql = """INSERT into camera_log (cam, valid, drct) VALUES
           ('%s', '%s', %s)""" % (cid, now.strftime("%Y-%m-%d %H:%I"),
           drct)
  mesosite.query( sql )

  sql = "DELETE from camera_current WHERE cam = '%s'" % (cid,)
  mesosite.query(sql)

  sql = "INSERT into camera_current(cam, valid, drct) values ('%s','%s',%s)" \
   %(cid, now.strftime('%Y-%m-%d %H:%M'), drct)
  mesosite.query(sql)

<|endoftext|>


<commit_before># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

import warnings
import copy
from pymatgen.core.structure import Molecule
from pymatgen.analysis.graphs import MoleculeGraph
from monty.dev import requires

try:
    from openbabel import openbabel as ob
    from openbabel import pybel as pb
except Exception:
    ob = None

"""
OpenBabel interface module, which opens up access to the hundreds of file
formats supported by OpenBabel. Requires openbabel with python bindings to be
installed. Please consult the
`openbabel documentation <http://openbabel.org/wiki/Main_Page>`_.
"""

__author__ = "Shyue Ping Ong, Qi Wang"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyuep@gmail.com"
__date__ = "Apr 28, 2012"

class BabelMolAdaptor:
    """
    Adaptor serves as a bridge between OpenBabel's Molecule and pymatgen's
    Molecule.
    """

    @requires(ob,
              "BabelMolAdaptor requires openbabel to be installed with "
              "Python bindings. Please get it at http://openbabel.org "
              "(version >=3.0.0).")
    def __init__(self, mol):
        """
        Initializes with pymatgen Molecule or OpenBabel"s OBMol.

        Args:
            mol: pymatgen's Molecule or OpenBabel OBMol
        """
        if isinstance(mol, Molecule):
            if not mol.is_ordered:
                raise ValueError("OpenBabel Molecule only supports ordered "
                                 "molecules.")

            # For some reason, manually adding atoms does not seem to create
            # the correct OBMol representation to do things like force field
            # optimization. So we go through the indirect route of creating
            # an XYZ file and reading in that file.
            obmol = ob.OBMol()
            obmol.BeginModify()
            for site in mol:
                coords = [c for c in site.coords]
                atomno = site.specie.Z
                obatom = ob.OBAtom()
                obatom.thisown = 0
                obatom.SetAtomicNum(atomno)
                obatom.SetVector(*coords)
                obmol.AddAtom(obatom)
                del obatom
            obmol.ConnectTheDots()
            obmol.PerceiveBondOrders()
            obmol.SetTotalSpinMultiplicity(mol.spin_multiplicity)
            obmol.SetTotalCharge(int(mol.charge))
            obmol.Center()
            obmol.EndModify()
            self._obmol = obmol
        elif isinstance(mol, ob.OBMol):
            self._obmol = mol

    @property
    def pymatgen_mol(self):
        """
        Returns pymatgen Molecule object.
        """
        sp = []
        coords = []
        for atom in ob.OBMolAtomIter(self._obmol):
            sp.append(atom.GetAtomicNum())
            coords.append([atom.GetX(), atom.GetY(), atom.GetZ()])
        return Molecule(sp, coords)

    @property
    def openbabel_mol(self):
        """
        Returns OpenBabel's OBMol.
        """
        return self._obmol

    def localopt(self, forcefield='mmff94', steps=500):
        """
        A wrapper to pybel's localopt method to optimize a Molecule.

        Args:
            forcefield: Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            steps: Default is 500.
        """
        pbmol = pb.Molecule(self._obmol)
        pbmol.localopt(forcefield=forcefield, steps=steps)
        self._obmol = pbmol.OBMol

    def make3d(self, forcefield="mmff94", steps=50):
        """
        A wrapper to pybel's make3D method generate a 3D structure from a
        2D or 0D structure.
        The 3D structure is made very quickly using a combination of rules
        (e.g. sp3 atoms should have four bonds arranged in a tetrahedron) and
        ring templates (e.g. cyclohexane is shaped like a chair). Once 3D
        coordinates are generated, hydrogens are added and a quick local
        optimization is carried out as default.

        The generated 3D structure can have clashes or have high energy
        structures due to some strain. Please consider to use the conformer
        search or geometry optimization to further optimize the structure.

        Args:
            forcefield: Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            steps: Default is 50.
        """
        pbmol = pb.Molecule(self._obmol)
        pbmol.make3D(forcefield=forcefield, steps=steps)
        self._obmol = pbmol.OBMol

    def add_hydrogen(self):
        """
        Add hydrogens (make all hydrogen explicit).
        """
        self._obmol.AddHydrogens()

    def remove_bond(self, idx1, idx2):
        """
        Remove a bond from an openbabel molecule

        Args:
            idx1: The atom index of one of the atoms participating the in bond
            idx2: The atom index of the other atom participating in the bond
        """
        for obbond in ob.OBMolBondIter(self._obmol):
            if (obbond.GetBeginAtomIdx() == idx1 and obbond.GetEndAtomIdx() == idx2) or (
                    obbond.GetBeginAtomIdx() == idx2 and obbond.GetEndAtomIdx() == idx1):
                self._obmol.DeleteBond(obbond)

    def rotor_conformer(self, *rotor_args, algo="WeightedRotorSearch",
                        forcefield="mmff94"):
        """
        Conformer search based on several Rotor Search algorithms of openbabel.
        If the input molecule is not 3D, make3d will be called (generate 3D
        structure, add hydrogen, a quick localopt). All hydrogen atoms need
        to be made explicit.

        Args:
            rotor_args: pass args to Rotor Search in openbabel.
                for "WeightedRotorSearch": (conformers, geomSteps,
                sampleRingBonds-default False)
                for "SystematicRotorSearch": (geomSteps-default 2500,
                sampleRingBonds-default False)
                for "RandomRotorSearch": (conformers, geomSteps-default 2500,
                sampleRingBonds-default False)
            algo (str): Default is "WeightedRotorSearch". Options are
                "SystematicRotorSearch", "RandomRotorSearch", and
                "WeightedRotorSearch".
            forcefield (str): Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
        """
        if self._obmol.GetDimension() != 3:
            self.make3d()
        else:
            self.add_hydrogen()

        ff = ob.OBForceField_FindType(forcefield)
        if ff == 0:
            warnings.warn("This input forcefield {} is not supported "
                          "in openbabel. The forcefield will be reset as "
                          "default 'mmff94' for now.".format(forcefield))
            ff = ob.OBForceField_FindType("mmff94")

        try:
            rotor_search = getattr(ff, algo)
        except AttributeError:
            warnings.warn("This input conformer search algorithm {} is not "
                          "supported in openbabel. Options are "
                          "'SystematicRotorSearch', 'RandomRotorSearch' "
                          "and 'WeightedRotorSearch'. "
                          "The algorithm will be reset as default "
                          "'WeightedRotorSearch' for now.".format(algo))
            rotor_search = ff.WeightedRotorSearch
        rotor_search(*rotor_args)
        ff.GetConformers(self._obmol)

    def gen3d_conformer(self):
        """
        A combined method to first generate 3D structures from 0D or 2D
        structures and then find the minimum energy conformer:

        1. Use OBBuilder to create a 3D structure using rules and ring templates
        2. Do 250 steps of a steepest descent geometry optimization with the
           MMFF94 forcefield
        3. Do 200 iterations of a Weighted Rotor conformational search
           (optimizing each conformer with 25 steps of a steepest descent)
        4. Do 250 steps of a conjugate gradient geometry optimization.

        Warning from openbabel docs:
        For many applications where 100s if not 1000s of molecules need to be
        processed, gen3d is rather SLOW. Sometimes this function can cause a
        segmentation fault.
        A future version of Open Babel will provide options for slow/medium/fast
        3D structure generation which will involve different compromises
        between speed and finding the global energy minimum.
        """
        gen3d = ob.OBOp.FindType("Gen3D")
        gen3d.Do(self._obmol)

    def confab_conformers(self, forcefield="mmff94", freeze_atoms=None,
                          rmsd_cutoff=0.5, energy_cutoff=50.0,
                          conf_cutoff=100000, verbose=False):
        """
        Conformer generation based on Confab to generate all diverse low-energy
        conformers for molecules. This is different from rotor_conformer or
        gen3d_conformer as it aims to not simply to find a low energy
        conformation but to generate several different conformations.

        Args:
            forcefield (str): Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            freeze_atoms ([int]): index of atoms to be freezed when performing
                conformer search, default is None.
            rmsd_cutoff (float): rmsd_cufoff, default is 0.5 Angstrom.
            energy_cutoff (float): energy_cutoff, default is 50.0 kcal/mol.
            conf_cutoff (float): max number of conformers to test,
                default is 1 million.
            verbose (bool): whether to display information on torsions found,
                default is False.

        Returns:
             (list): list of pymatgen Molecule objects for generated conformers.
        """
        if self._obmol.GetDimension() != 3:
            self.make3d()
        else:
            self.add_hydrogen()

        ff = ob.OBForceField_FindType(forcefield)
        if ff == 0:
            print("Could not find forcefield {} in openbabel, the forcefield "
                  "will be reset as default 'mmff94'".format(forcefield))
            ff = ob.OBForceField_FindType("mmff94")

        if freeze_atoms:
            print('{} atoms will be freezed'.format(len(freeze_atoms)))
            constraints = ob.OBFFConstraints()

            for atom in ob.OBMolAtomIter(self._obmol):
                atom_id = atom.GetIndex() + 1
                if id in freeze_atoms:
                    constraints.AddAtomConstraint(atom_id)
            ff.SetConstraints(constraints)

        # Confab conformer generation
        ff.DiverseConfGen(rmsd_cutoff, conf_cutoff, energy_cutoff,
                          verbose)
        ff.GetConformers(self._obmol)

        # Number of conformers generated by Confab conformer generation
        conformer_num = self._obmol.NumConformers()

        conformers = []
        for i in range(conformer_num):
            self._obmol.SetConformer(i)
            conformer = copy.deepcopy(BabelMolAdaptor(self._obmol).pymatgen_mol)
            conformers.append(conformer)
        self._obmol.SetConformer(0)
        return conformers

    @property
    def pybel_mol(self):
        """
        Returns Pybel's Molecule object.
        """
        return pb.Molecule(self._obmol)

    def write_file(self, filename, file_format="xyz"):
        """
        Uses OpenBabel to output all supported formats.

        Args:
            filename: Filename of file to output
            file_format: String specifying any OpenBabel supported formats.
        """
        mol = pb.Molecule(self._obmol)
        return mol.write(file_format, filename, overwrite=True)

    @staticmethod
    def from_file(filename, file_format="xyz", return_all_molecules=False):
        """
        Uses OpenBabel to read a molecule from a file in all supported formats.

        Args:
            filename: Filename of input file
            file_format: String specifying any OpenBabel supported formats.
            return_all_molecules: If ``True``, will return a list of
                ``BabelMolAdaptor`` instances, one for each molecule found in
                the file. If ``False``, will return only the first molecule.

        Returns:
            BabelMolAdaptor object or list thereof
        """
        mols = pb.readfile(str(file_format), str(filename))
        if return_all_molecules:
            return [BabelMolAdaptor(mol.OBMol) for mol in mols]
        else:
            return BabelMolAdaptor(next(mols).OBMol)

    @staticmethod
    def from_molecule_graph(mol):
        """
        Read a molecule from a pymatgen MoleculeGraph object.

        Args:
            mol: pymatgen MoleculeGraph object.

        Returns:
            BabelMolAdaptor object
        """
        if isinstance(mol, MoleculeGraph):
            return BabelMolAdaptor(mol.molecule)

    @staticmethod
    def from_string(string_data, file_format="xyz"):
        """
        Uses OpenBabel to read a molecule from a string in all supported
        formats.

        Args:
            string_data: String containing molecule data.
            file_format: String specifying any OpenBabel supported formats.

        Returns:
            BabelMolAdaptor object
        """
        mols = pb.readstring(str(file_format), str(string_data))
        return BabelMolAdaptor(mols.OBMol)
<commit_msg>Fix missing module doc in babel interface.<commit_after># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

"""
OpenBabel interface module, which opens up access to the hundreds of file
formats supported by OpenBabel. Requires openbabel with python bindings to be
installed. Please consult the
`openbabel documentation <http://openbabel.org/wiki/Main_Page>`_.
"""

import warnings
import copy
from pymatgen.core.structure import Molecule
from pymatgen.analysis.graphs import MoleculeGraph
from monty.dev import requires

try:
    from openbabel import openbabel as ob
    from openbabel import pybel as pb
except Exception:
    ob = None

__author__ = "Shyue Ping Ong, Qi Wang"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyuep@gmail.com"
__date__ = "Apr 28, 2012"

class BabelMolAdaptor:
    """
    Adaptor serves as a bridge between OpenBabel's Molecule and pymatgen's
    Molecule.
    """

    @requires(ob,
              "BabelMolAdaptor requires openbabel to be installed with "
              "Python bindings. Please get it at http://openbabel.org "
              "(version >=3.0.0).")
    def __init__(self, mol):
        """
        Initializes with pymatgen Molecule or OpenBabel"s OBMol.

        Args:
            mol: pymatgen's Molecule or OpenBabel OBMol
        """
        if isinstance(mol, Molecule):
            if not mol.is_ordered:
                raise ValueError("OpenBabel Molecule only supports ordered "
                                 "molecules.")

            # For some reason, manually adding atoms does not seem to create
            # the correct OBMol representation to do things like force field
            # optimization. So we go through the indirect route of creating
            # an XYZ file and reading in that file.
            obmol = ob.OBMol()
            obmol.BeginModify()
            for site in mol:
                coords = [c for c in site.coords]
                atomno = site.specie.Z
                obatom = ob.OBAtom()
                obatom.thisown = 0
                obatom.SetAtomicNum(atomno)
                obatom.SetVector(*coords)
                obmol.AddAtom(obatom)
                del obatom
            obmol.ConnectTheDots()
            obmol.PerceiveBondOrders()
            obmol.SetTotalSpinMultiplicity(mol.spin_multiplicity)
            obmol.SetTotalCharge(int(mol.charge))
            obmol.Center()
            obmol.EndModify()
            self._obmol = obmol
        elif isinstance(mol, ob.OBMol):
            self._obmol = mol

    @property
    def pymatgen_mol(self):
        """
        Returns pymatgen Molecule object.
        """
        sp = []
        coords = []
        for atom in ob.OBMolAtomIter(self._obmol):
            sp.append(atom.GetAtomicNum())
            coords.append([atom.GetX(), atom.GetY(), atom.GetZ()])
        return Molecule(sp, coords)

    @property
    def openbabel_mol(self):
        """
        Returns OpenBabel's OBMol.
        """
        return self._obmol

    def localopt(self, forcefield='mmff94', steps=500):
        """
        A wrapper to pybel's localopt method to optimize a Molecule.

        Args:
            forcefield: Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            steps: Default is 500.
        """
        pbmol = pb.Molecule(self._obmol)
        pbmol.localopt(forcefield=forcefield, steps=steps)
        self._obmol = pbmol.OBMol

    def make3d(self, forcefield="mmff94", steps=50):
        """
        A wrapper to pybel's make3D method generate a 3D structure from a
        2D or 0D structure.
        The 3D structure is made very quickly using a combination of rules
        (e.g. sp3 atoms should have four bonds arranged in a tetrahedron) and
        ring templates (e.g. cyclohexane is shaped like a chair). Once 3D
        coordinates are generated, hydrogens are added and a quick local
        optimization is carried out as default.

        The generated 3D structure can have clashes or have high energy
        structures due to some strain. Please consider to use the conformer
        search or geometry optimization to further optimize the structure.

        Args:
            forcefield: Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            steps: Default is 50.
        """
        pbmol = pb.Molecule(self._obmol)
        pbmol.make3D(forcefield=forcefield, steps=steps)
        self._obmol = pbmol.OBMol

    def add_hydrogen(self):
        """
        Add hydrogens (make all hydrogen explicit).
        """
        self._obmol.AddHydrogens()

    def remove_bond(self, idx1, idx2):
        """
        Remove a bond from an openbabel molecule

        Args:
            idx1: The atom index of one of the atoms participating the in bond
            idx2: The atom index of the other atom participating in the bond
        """
        for obbond in ob.OBMolBondIter(self._obmol):
            if (obbond.GetBeginAtomIdx() == idx1 and obbond.GetEndAtomIdx() == idx2) or (
                    obbond.GetBeginAtomIdx() == idx2 and obbond.GetEndAtomIdx() == idx1):
                self._obmol.DeleteBond(obbond)

    def rotor_conformer(self, *rotor_args, algo="WeightedRotorSearch",
                        forcefield="mmff94"):
        """
        Conformer search based on several Rotor Search algorithms of openbabel.
        If the input molecule is not 3D, make3d will be called (generate 3D
        structure, add hydrogen, a quick localopt). All hydrogen atoms need
        to be made explicit.

        Args:
            rotor_args: pass args to Rotor Search in openbabel.
                for "WeightedRotorSearch": (conformers, geomSteps,
                sampleRingBonds-default False)
                for "SystematicRotorSearch": (geomSteps-default 2500,
                sampleRingBonds-default False)
                for "RandomRotorSearch": (conformers, geomSteps-default 2500,
                sampleRingBonds-default False)
            algo (str): Default is "WeightedRotorSearch". Options are
                "SystematicRotorSearch", "RandomRotorSearch", and
                "WeightedRotorSearch".
            forcefield (str): Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
        """
        if self._obmol.GetDimension() != 3:
            self.make3d()
        else:
            self.add_hydrogen()

        ff = ob.OBForceField_FindType(forcefield)
        if ff == 0:
            warnings.warn("This input forcefield {} is not supported "
                          "in openbabel. The forcefield will be reset as "
                          "default 'mmff94' for now.".format(forcefield))
            ff = ob.OBForceField_FindType("mmff94")

        try:
            rotor_search = getattr(ff, algo)
        except AttributeError:
            warnings.warn("This input conformer search algorithm {} is not "
                          "supported in openbabel. Options are "
                          "'SystematicRotorSearch', 'RandomRotorSearch' "
                          "and 'WeightedRotorSearch'. "
                          "The algorithm will be reset as default "
                          "'WeightedRotorSearch' for now.".format(algo))
            rotor_search = ff.WeightedRotorSearch
        rotor_search(*rotor_args)
        ff.GetConformers(self._obmol)

    def gen3d_conformer(self):
        """
        A combined method to first generate 3D structures from 0D or 2D
        structures and then find the minimum energy conformer:

        1. Use OBBuilder to create a 3D structure using rules and ring templates
        2. Do 250 steps of a steepest descent geometry optimization with the
           MMFF94 forcefield
        3. Do 200 iterations of a Weighted Rotor conformational search
           (optimizing each conformer with 25 steps of a steepest descent)
        4. Do 250 steps of a conjugate gradient geometry optimization.

        Warning from openbabel docs:
        For many applications where 100s if not 1000s of molecules need to be
        processed, gen3d is rather SLOW. Sometimes this function can cause a
        segmentation fault.
        A future version of Open Babel will provide options for slow/medium/fast
        3D structure generation which will involve different compromises
        between speed and finding the global energy minimum.
        """
        gen3d = ob.OBOp.FindType("Gen3D")
        gen3d.Do(self._obmol)

    def confab_conformers(self, forcefield="mmff94", freeze_atoms=None,
                          rmsd_cutoff=0.5, energy_cutoff=50.0,
                          conf_cutoff=100000, verbose=False):
        """
        Conformer generation based on Confab to generate all diverse low-energy
        conformers for molecules. This is different from rotor_conformer or
        gen3d_conformer as it aims to not simply to find a low energy
        conformation but to generate several different conformations.

        Args:
            forcefield (str): Default is mmff94. Options are 'gaff', 'ghemical',
                'mmff94', 'mmff94s', and 'uff'.
            freeze_atoms ([int]): index of atoms to be freezed when performing
                conformer search, default is None.
            rmsd_cutoff (float): rmsd_cufoff, default is 0.5 Angstrom.
            energy_cutoff (float): energy_cutoff, default is 50.0 kcal/mol.
            conf_cutoff (float): max number of conformers to test,
                default is 1 million.
            verbose (bool): whether to display information on torsions found,
                default is False.

        Returns:
             (list): list of pymatgen Molecule objects for generated conformers.
        """
        if self._obmol.GetDimension() != 3:
            self.make3d()
        else:
            self.add_hydrogen()

        ff = ob.OBForceField_FindType(forcefield)
        if ff == 0:
            print("Could not find forcefield {} in openbabel, the forcefield "
                  "will be reset as default 'mmff94'".format(forcefield))
            ff = ob.OBForceField_FindType("mmff94")

        if freeze_atoms:
            print('{} atoms will be freezed'.format(len(freeze_atoms)))
            constraints = ob.OBFFConstraints()

            for atom in ob.OBMolAtomIter(self._obmol):
                atom_id = atom.GetIndex() + 1
                if id in freeze_atoms:
                    constraints.AddAtomConstraint(atom_id)
            ff.SetConstraints(constraints)

        # Confab conformer generation
        ff.DiverseConfGen(rmsd_cutoff, conf_cutoff, energy_cutoff,
                          verbose)
        ff.GetConformers(self._obmol)

        # Number of conformers generated by Confab conformer generation
        conformer_num = self._obmol.NumConformers()

        conformers = []
        for i in range(conformer_num):
            self._obmol.SetConformer(i)
            conformer = copy.deepcopy(BabelMolAdaptor(self._obmol).pymatgen_mol)
            conformers.append(conformer)
        self._obmol.SetConformer(0)
        return conformers

    @property
    def pybel_mol(self):
        """
        Returns Pybel's Molecule object.
        """
        return pb.Molecule(self._obmol)

    def write_file(self, filename, file_format="xyz"):
        """
        Uses OpenBabel to output all supported formats.

        Args:
            filename: Filename of file to output
            file_format: String specifying any OpenBabel supported formats.
        """
        mol = pb.Molecule(self._obmol)
        return mol.write(file_format, filename, overwrite=True)

    @staticmethod
    def from_file(filename, file_format="xyz", return_all_molecules=False):
        """
        Uses OpenBabel to read a molecule from a file in all supported formats.

        Args:
            filename: Filename of input file
            file_format: String specifying any OpenBabel supported formats.
            return_all_molecules: If ``True``, will return a list of
                ``BabelMolAdaptor`` instances, one for each molecule found in
                the file. If ``False``, will return only the first molecule.

        Returns:
            BabelMolAdaptor object or list thereof
        """
        mols = pb.readfile(str(file_format), str(filename))
        if return_all_molecules:
            return [BabelMolAdaptor(mol.OBMol) for mol in mols]
        else:
            return BabelMolAdaptor(next(mols).OBMol)

    @staticmethod
    def from_molecule_graph(mol):
        """
        Read a molecule from a pymatgen MoleculeGraph object.

        Args:
            mol: pymatgen MoleculeGraph object.

        Returns:
            BabelMolAdaptor object
        """
        if isinstance(mol, MoleculeGraph):
            return BabelMolAdaptor(mol.molecule)

    @staticmethod
    def from_string(string_data, file_format="xyz"):
        """
        Uses OpenBabel to read a molecule from a string in all supported
        formats.

        Args:
            string_data: String containing molecule data.
            file_format: String specifying any OpenBabel supported formats.

        Returns:
            BabelMolAdaptor object
        """
        mols = pb.readstring(str(file_format), str(string_data))
        return BabelMolAdaptor(mols.OBMol)
<|endoftext|>