<commit_before># coding:utf-8
from sqlalchemy import text
from db.basic_db import db_session
from db.models import SeedIds
from decorators.decorator import db_commit_decorator

def get_seed_ids():
    return db_session.query(SeedIds.uid).filter(text('is_crawled=0')).all()

@db_commit_decorator
def set_seed_crawled(uid, result):
    """
    该表适用于用户抓取相关逻辑
    :param uid: 被抓取用户id
    :param result: 抓取结果
    :return: None
    """
    seed = db_session.query(SeedIds).filter(SeedIds.uid == uid).first()
    if seed:
        if seed.is_crawled == 0:
            seed.is_crawled = result
    else:
        seed = SeedIds(uid=uid, is_crawled=result)
        db_session.add(seed)
    db_session.commit()

def get_seed_by_id(uid):
    return db_session.query(SeedIds).filter(SeedIds.uid == uid).first()

@db_commit_decorator
def insert_seeds(ids):
    # 批量插入，遇到重复则跳过
    db_session.execute(SeedIds.__table__.insert().prefix_with('IGNORE'), [{'uid': i} for i in ids])
    db_session.commit()

@db_commit_decorator
def set_seed_other_crawled(uid):
    """
    存在则更新，不存在则插入
    :param uid: 用户id
    :return: None
    """
    seed = get_seed_by_id(uid)
    if seed is None:
        seed = SeedIds(uid=uid, is_crawled=1, other_crawled=1)
        db_session.add(seed)
    else:
        seed.other_crawled = 1
    db_session.commit()
<commit_msg>增加seed_ids表的操作<commit_after># coding:utf-8
from sqlalchemy import text
from db.basic_db import db_session
from db.models import SeedIds
from decorators.decorator import db_commit_decorator

def get_seed_ids():
    """
    获取所有个人信息需要被抓取的用户id
    :return:
    """
    return db_session.query(SeedIds.uid).filter(text('is_crawled=0')).all()

def get_home_ids():
    """
    获取所有主页需要被抓取的用户id
    :return:
    """
    return db_session.query(SeedIds.uid).filter(text('home_crawled=0')).all()

@db_commit_decorator
def set_seed_crawled(uid, result):
    """
    该表适用于用户抓取相关逻辑
    :param uid: 被抓取用户id
    :param result: 抓取结果
    :return: None
    """
    seed = db_session.query(SeedIds).filter(SeedIds.uid == uid).first()
    if seed:
        if seed.is_crawled == 0:
            seed.is_crawled = result
    else:
        seed = SeedIds(uid=uid, is_crawled=result)
        db_session.add(seed)
    db_session.commit()

def get_seed_by_id(uid):
    return db_session.query(SeedIds).filter(SeedIds.uid == uid).first()

@db_commit_decorator
def insert_seeds(ids):
    # 批量插入，遇到重复则跳过
    db_session.execute(SeedIds.__table__.insert().prefix_with('IGNORE'), [{'uid': i} for i in ids])
    db_session.commit()

@db_commit_decorator
def set_seed_other_crawled(uid):
    """
    存在则更新，不存在则插入
    :param uid: 用户id
    :return: None
    """
    seed = get_seed_by_id(uid)
    if seed is None:
        seed = SeedIds(uid=uid, is_crawled=1, other_crawled=1)
        db_session.add(seed)
    else:
        seed.other_crawled = 1
    db_session.commit()

@db_commit_decorator
def set_seed_home_crawled(uid):
    """
    将该用户的home_crawled字段设置为已经抓取,它和set_seed_other_crawled(uid)不同，因为后者的uid是通过用户关注
    或者粉丝列表获取的,它们的信息有可能是没有的，所以需要判断是否存在，而这里是基于已经存在的信息
    :param uid: 用户id
    :return: None
    """
    seed = get_seed_by_id(uid)
    seed.home_crawled = 1
    db_session.commit()
<|endoftext|>


<commit_before>

import sys
import os

# from casa_tools import myclean
from tasks import clean, feather, rmtables

'''
Cleans an MS with a single channel given a mask and a model
'''

vis = sys.argv[-3]
model = sys.argv[-2]
mask = sys.argv[-1]

if model == "None":
    model = None
if mask == "None":
    mask = None

out_root = vis[:-3]

# Define a valid mask. This stays true so long as a given mask contains a
# usable region.
valid_mask = True
if mask is not None:
    # Check if there's anything in the mask before cleaning
    ia.open(mask)
    stats_dict = ia.statistics()
    ia.close()
    # If there's nothing there, max == min
    max_val = stats_dict["max"]
    if max_val == 0:
        casalog.post("No valid region in the given mask. Skipping clean.")
        valid_mask = False
    else:
        casalog.post("Mask contains valid region. Proceeding with clean.")

if valid_mask:
    clean(vis=vis, imagename=out_root + '.clean', field='M33*',
          restfreq='1420.40575177MHz',
          mode='channel', width=1, nchan=1, start=1,
          cell='3arcsec', multiscale=[0, 4, 8, 20, 40, 80],
          threshold='1.8mJy/beam', imagermode='mosaic', gain=0.1,
          imsize=[2560, 2560], weighting='natural', robust=0.0, niter=200000,
          pbcor=True, minpb=0.2, interpolation='linear', usescratch=False,
          phasecenter='J2000 01h33m50.904 +30d39m35.79', veltype='radio',
          modelimage=model, mask=mask)

    # Only run if the image was actually produces
    if os.path.exists(out_root + ".clean.image"):
        feather(imagename=out_root + ".clean.image.feathered",
                highres=out_root + ".clean.image",
                lowres=model)

    # If something went awry, and the image wasn't produced, remove the
    # remnants.
    if not os.path.exists(out_root + ".clean.image"):
        rmtables(out_root + "clean.*")
<commit_msg>One more<commit_after>

import sys
import os

# from casa_tools import myclean
from tasks import clean, feather, rmtables

'''
Cleans an MS with a single channel given a mask and a model
'''

vis = sys.argv[-3]
model = sys.argv[-2]
mask = sys.argv[-1]

if model == "None":
    model = None
if mask == "None":
    mask = None

out_root = vis[:-3]

# Define a valid mask. This stays true so long as a given mask contains a
# usable region.
valid_mask = True
if mask is not None:
    # Check if there's anything in the mask before cleaning
    ia.open(mask)
    stats_dict = ia.statistics()
    ia.close()
    # If there's nothing there, max == min
    max_val = stats_dict["max"]
    if max_val == 0:
        casalog.post("No valid region in the given mask. Skipping clean.")
        valid_mask = False
    else:
        casalog.post("Mask contains valid region. Proceeding with clean.")

if valid_mask:
    clean(vis=vis, imagename=out_root + '.clean', field='M33*',
          restfreq='1420.40575177MHz',
          mode='channel', width=1, nchan=1, start=1,
          cell='3arcsec', multiscale=[0, 4, 8, 20, 40, 80],
          threshold='1.8mJy/beam', imagermode='mosaic', gain=0.1,
          imsize=[2560, 2560], weighting='natural', robust=0.0, niter=200000,
          pbcor=True, minpb=0.2, interpolation='linear', usescratch=False,
          phasecenter='J2000 01h33m50.904 +30d39m35.79', veltype='radio',
          modelimage=model, mask=mask)

    # Only run if the image was actually produces
    if os.path.exists(out_root + ".clean.image"):
        feather(imagename=out_root + ".clean.image.feathered",
                highres=out_root + ".clean.image",
                lowres=model)

    # If something went awry, and the image wasn't produced, remove the
    # remnants.
    if not os.path.exists(out_root + ".clean.image"):
        rmtables(out_root + ".clean.*")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tool to bundle root filesystem to a tarball.

Creates a tar bundle and a Manifest, which can be uploaded to image store.
"""

import logging
from optparse import OptionParser
import os
import shutil
import subprocess
import tempfile
import time

from gcimagebundlelib import block_disk
from gcimagebundlelib import exclude_spec
from gcimagebundlelib import platform_factory
from gcimagebundlelib import utils

def SetupArgsParser():
  """Sets up the command line flags."""
  parser = OptionParser()
  parser.add_option('-d', '--disk', dest='disk',
                    default='/dev/sda',
                    help='Disk to bundle.')
  parser.add_option('-r', '--root', dest='root_directory',
                    default='/', metavar='ROOT',
                    help='Root of the file system to bundle.'
                    ' Recursively bundles all sub directories.')
  parser.add_option('-e', '--excludes', dest='excludes',
                    help='Comma separated list of sub directories to exclude.'
                    ' The defaults are platform specific.')
  parser.add_option('-o', '--output_directory', dest='output_directory',
                    default='/tmp/', metavar='DIR',
                    help='Output directory for image.')
  parser.add_option('--output_file_name', dest='output_file_name',
                    default=None, metavar='FILENAME',
                    help=('Output filename for the image. Default is a digest'
                          ' of the image bytes.'))
  parser.add_option('--include_mounts', dest='include_mounts',
                    help='Don\'t ignore mounted filesystems under ROOT.',
                    action='store_true', default=False)
  parser.add_option('-v', '--version',
                    action='store_true', dest='display_version', default=False,
                    help='Print the tool version.')
  parser.add_option('--loglevel', dest='log_level',
                    help='Debug logging level.', default='INFO',
                    choices=['DEBUG', 'INFO', 'WARNING', 'ERROR' 'CRITICAL'])
  parser.add_option('--log_file', dest='log_file',
                    help='Output file for log messages.')
  parser.add_option('-k', '--key', dest='key', default='nebula',
                    help='Public key used for signing the image.')
  parser.add_option('--nocleanup', dest='cleanup',
                    action='store_false', default=True,
                    help=' Do not clean up temporary and log files.')
  #TODO(user): Get dehumanize.
  parser.add_option('--fssize', dest='fs_size', default=10*1024*1024*1024,
                    type='int', help='File system size in bytes')
  parser.add_option('-b', '--bucket', dest='bucket',
                    help='Destination storage bucket')
  parser.add_option('-f', '--filesystem', dest='file_system',
                    default=None,
                    help='File system type for the image.')
  parser.add_option('--skip_disk_space_check', dest='skip_disk_space_check',
                    default=False, action='store_true',
                    help='Skip the disk space requirement check.')

  return parser

def VerifyArgs(parser, options):
  """Verifies that commandline flags are consistent."""
  if not options.output_directory:
    parser.error('output bundle directory must be specified.')
  if not os.path.exists(options.output_directory):
    parser.error('output bundle directory does not exist.')

  # TODO(user): add more verification as needed

def EnsureSuperUser():
  """Ensures that current user has super user privileges."""
  if os.getuid() != 0:
    logging.warning('Tool must be run as root.')
    exit(-1)

def GetLogLevel(options):
  """Log Level string to logging.LogLevel mapping."""
  level = {
      'DEBUG': logging.DEBUG,
      'INFO': logging.INFO,
      'WARNING': logging.WARNING,
      'ERROR': logging.ERROR,
      'CRITICAL': logging.CRITICAL
  }
  if options.log_level in level:
    return level[options.log_level]
  print 'Invalid logging level. defaulting to INFO.'
  return logging.INFO

def SetupLogging(options, log_dir='/tmp'):
  """Set up logging.

  All messages above INFO level are also logged to console.

  Args:
    options: collection of command line options.
    log_dir: directory used to generate log files.
  """
  if options.log_file:
    logfile = options.log_file
  else:
    logfile = tempfile.mktemp(dir=log_dir, prefix='bundle_log_')
  print 'Starting logging in %s' % logfile
  logging.basicConfig(filename=logfile,
                      level=GetLogLevel(options),
                      format='%(asctime)s %(levelname)s:%(name)s:%(message)s')
  # Use GMT timestamp in logging.
  logging.Formatter.converter=time.gmtime
  console = logging.StreamHandler()
  console.setLevel(GetLogLevel(options))
  logging.getLogger().addHandler(console)

def PrintVersionInfo():
  #TODO: Should read from the VERSION file instead.
  print 'version 1.2.9'

def GetTargetFilesystem(options, guest_platform):
  if options.file_system:
    return options.file_system
  else:
    return guest_platform.GetPreferredFilesystemType()

def main():
  parser = SetupArgsParser()
  (options, _) = parser.parse_args()
  if options.display_version:
    PrintVersionInfo()
    return 0
  EnsureSuperUser()
  VerifyArgs(parser, options)

  scratch_dir = tempfile.mkdtemp(dir=options.output_directory)
  SetupLogging(options, scratch_dir)
  try:
    guest_platform = platform_factory.PlatformFactory(
        options.root_directory).GetPlatform()
  except platform_factory.UnknownPlatformException:
    logging.critical('Platform is not supported.'
                     ' Platform rules can be added to platform_factory.py.')
    return -1

  temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz')

  file_system = GetTargetFilesystem(options, guest_platform)
  logging.info('File System: %s', file_system)
  logging.info('Disk Size: %s bytes', options.fs_size)
  bundle = block_disk.RootFsRaw(
      options.fs_size, file_system, options.skip_disk_space_check)
  bundle.SetTarfile(temp_file_name)
  if options.disk:
    readlink_command = ['readlink', '-f', options.disk]
    final_path = utils.RunCommand(readlink_command).strip()
    logging.info('Resolved %s to %s', options.disk, final_path)
    bundle.AddDisk(final_path)
    # TODO(user): Find the location where the first partition of the disk
    # is mounted and add it as the source instead of relying on the source
    # param flag
  bundle.AddSource(options.root_directory)
  bundle.SetKey(options.key)
  bundle.SetScratchDirectory(scratch_dir)

  # Merge platform specific exclude list, mounts points
  # and user specified excludes
  excludes = guest_platform.GetExcludeList()
  if options.excludes:
    excludes.extend([exclude_spec.ExcludeSpec(x) for x in
                     options.excludes.split(',')])
  logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes]))
  bundle.AppendExcludes(excludes)
  if not options.include_mounts:
    mount_points = utils.GetMounts(options.root_directory)
    logging.info('ignoring mounts %s', ' '.join(mount_points))
    bundle.AppendExcludes([exclude_spec.ExcludeSpec(x, preserve_dir=True) for x
                           in utils.GetMounts(options.root_directory)])
  bundle.SetPlatform(guest_platform)

  # Verify that bundle attributes are correct and create tar bundle.
  bundle.Verify()
  (fs_size, digest) = bundle.Bundleup()
  if not digest:
    logging.critical('Could not get digest for the bundle.'
                     ' The bundle may not be created correctly')
    return -1
  if fs_size > options.fs_size:
    logging.critical('Size of tar %d exceeds the file system size %d.', fs_size,
                     options.fs_size)
    return -1

  if options.output_file_name:
    output_file = os.path.join(
        options.output_directory, options.output_file_name)
  else:
    output_file = os.path.join(
        options.output_directory, '%s.image.tar.gz' % digest)

  os.rename(temp_file_name, output_file)
  logging.info('Created tar.gz file at %s' % output_file)

  if options.bucket:
    bucket = options.bucket
    if bucket.startswith('gs://'):
      output_bucket = '%s/%s' % (
          bucket, os.path.basename(output_file))
    else:
      output_bucket = 'gs://%s/%s' % (
          bucket, os.path.basename(output_file))

    # /usr/local/bin not in redhat root PATH by default
    if '/usr/local/bin' not in os.environ['PATH']:
      os.environ['PATH'] += ':/usr/local/bin'

    # TODO: Consider using boto library directly.
    cmd = ['gsutil', 'cp', output_file, output_bucket]
    retcode = subprocess.call(cmd)
    if retcode != 0:
      logging.critical('Failed to copy image to bucket. '
                       'gsutil returned %d. To retry, run the command: %s',
                       retcode, ' '.join(cmd))

      return -1
    logging.info('Uploaded image to %s', output_bucket)

  # If we've uploaded, then we can remove the local file.
  os.remove(output_file)

  if options.cleanup:
    shutil.rmtree(scratch_dir)
<commit_msg>Undid another indentation error<commit_after># -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tool to bundle root filesystem to a tarball.

Creates a tar bundle and a Manifest, which can be uploaded to image store.
"""

import logging
from optparse import OptionParser
import os
import shutil
import subprocess
import tempfile
import time

from gcimagebundlelib import block_disk
from gcimagebundlelib import exclude_spec
from gcimagebundlelib import platform_factory
from gcimagebundlelib import utils

def SetupArgsParser():
  """Sets up the command line flags."""
  parser = OptionParser()
  parser.add_option('-d', '--disk', dest='disk',
                    default='/dev/sda',
                    help='Disk to bundle.')
  parser.add_option('-r', '--root', dest='root_directory',
                    default='/', metavar='ROOT',
                    help='Root of the file system to bundle.'
                    ' Recursively bundles all sub directories.')
  parser.add_option('-e', '--excludes', dest='excludes',
                    help='Comma separated list of sub directories to exclude.'
                    ' The defaults are platform specific.')
  parser.add_option('-o', '--output_directory', dest='output_directory',
                    default='/tmp/', metavar='DIR',
                    help='Output directory for image.')
  parser.add_option('--output_file_name', dest='output_file_name',
                    default=None, metavar='FILENAME',
                    help=('Output filename for the image. Default is a digest'
                          ' of the image bytes.'))
  parser.add_option('--include_mounts', dest='include_mounts',
                    help='Don\'t ignore mounted filesystems under ROOT.',
                    action='store_true', default=False)
  parser.add_option('-v', '--version',
                    action='store_true', dest='display_version', default=False,
                    help='Print the tool version.')
  parser.add_option('--loglevel', dest='log_level',
                    help='Debug logging level.', default='INFO',
                    choices=['DEBUG', 'INFO', 'WARNING', 'ERROR' 'CRITICAL'])
  parser.add_option('--log_file', dest='log_file',
                    help='Output file for log messages.')
  parser.add_option('-k', '--key', dest='key', default='nebula',
                    help='Public key used for signing the image.')
  parser.add_option('--nocleanup', dest='cleanup',
                    action='store_false', default=True,
                    help=' Do not clean up temporary and log files.')
  #TODO(user): Get dehumanize.
  parser.add_option('--fssize', dest='fs_size', default=10*1024*1024*1024,
                    type='int', help='File system size in bytes')
  parser.add_option('-b', '--bucket', dest='bucket',
                    help='Destination storage bucket')
  parser.add_option('-f', '--filesystem', dest='file_system',
                    default=None,
                    help='File system type for the image.')
  parser.add_option('--skip_disk_space_check', dest='skip_disk_space_check',
                    default=False, action='store_true',
                    help='Skip the disk space requirement check.')

  return parser

def VerifyArgs(parser, options):
  """Verifies that commandline flags are consistent."""
  if not options.output_directory:
    parser.error('output bundle directory must be specified.')
  if not os.path.exists(options.output_directory):
    parser.error('output bundle directory does not exist.')

  # TODO(user): add more verification as needed

def EnsureSuperUser():
  """Ensures that current user has super user privileges."""
  if os.getuid() != 0:
    logging.warning('Tool must be run as root.')
    exit(-1)

def GetLogLevel(options):
  """Log Level string to logging.LogLevel mapping."""
  level = {
      'DEBUG': logging.DEBUG,
      'INFO': logging.INFO,
      'WARNING': logging.WARNING,
      'ERROR': logging.ERROR,
      'CRITICAL': logging.CRITICAL
  }
  if options.log_level in level:
    return level[options.log_level]
  print 'Invalid logging level. defaulting to INFO.'
  return logging.INFO

def SetupLogging(options, log_dir='/tmp'):
  """Set up logging.

  All messages above INFO level are also logged to console.

  Args:
    options: collection of command line options.
    log_dir: directory used to generate log files.
  """
  if options.log_file:
    logfile = options.log_file
  else:
    logfile = tempfile.mktemp(dir=log_dir, prefix='bundle_log_')
  print 'Starting logging in %s' % logfile
  logging.basicConfig(filename=logfile,
                      level=GetLogLevel(options),
                      format='%(asctime)s %(levelname)s:%(name)s:%(message)s')
  # Use GMT timestamp in logging.
  logging.Formatter.converter=time.gmtime
  console = logging.StreamHandler()
  console.setLevel(GetLogLevel(options))
  logging.getLogger().addHandler(console)

def PrintVersionInfo():
  #TODO: Should read from the VERSION file instead.
  print 'version 1.2.9'

def GetTargetFilesystem(options, guest_platform):
  if options.file_system:
    return options.file_system
  else:
    return guest_platform.GetPreferredFilesystemType()

def main():
  parser = SetupArgsParser()
  (options, _) = parser.parse_args()
  if options.display_version:
    PrintVersionInfo()
    return 0
  EnsureSuperUser()
  VerifyArgs(parser, options)

  scratch_dir = tempfile.mkdtemp(dir=options.output_directory)
  SetupLogging(options, scratch_dir)
  try:
    guest_platform = platform_factory.PlatformFactory(
        options.root_directory).GetPlatform()
  except platform_factory.UnknownPlatformException:
    logging.critical('Platform is not supported.'
                     ' Platform rules can be added to platform_factory.py.')
    return -1

  temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz')

  file_system = GetTargetFilesystem(options, guest_platform)
  logging.info('File System: %s', file_system)
  logging.info('Disk Size: %s bytes', options.fs_size)
  bundle = block_disk.RootFsRaw(
      options.fs_size, file_system, options.skip_disk_space_check)
  bundle.SetTarfile(temp_file_name)
  if options.disk:
    readlink_command = ['readlink', '-f', options.disk]
    final_path = utils.RunCommand(readlink_command).strip()
    logging.info('Resolved %s to %s', options.disk, final_path)
    bundle.AddDisk(final_path)
    # TODO(user): Find the location where the first partition of the disk
    # is mounted and add it as the source instead of relying on the source
    # param flag
  bundle.AddSource(options.root_directory)
  bundle.SetKey(options.key)
  bundle.SetScratchDirectory(scratch_dir)

  # Merge platform specific exclude list, mounts points
  # and user specified excludes
  excludes = guest_platform.GetExcludeList()
  if options.excludes:
    excludes.extend([exclude_spec.ExcludeSpec(x) for x in
                     options.excludes.split(',')])
  logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes]))
  bundle.AppendExcludes(excludes)
  if not options.include_mounts:
    mount_points = utils.GetMounts(options.root_directory)
    logging.info('ignoring mounts %s', ' '.join(mount_points))
    bundle.AppendExcludes([exclude_spec.ExcludeSpec(x, preserve_dir=True) for x
                           in utils.GetMounts(options.root_directory)])
  bundle.SetPlatform(guest_platform)

  # Verify that bundle attributes are correct and create tar bundle.
  bundle.Verify()
  (fs_size, digest) = bundle.Bundleup()
  if not digest:
    logging.critical('Could not get digest for the bundle.'
                     ' The bundle may not be created correctly')
    return -1
  if fs_size > options.fs_size:
    logging.critical('Size of tar %d exceeds the file system size %d.', fs_size,
                     options.fs_size)
    return -1

  if options.output_file_name:
    output_file = os.path.join(
        options.output_directory, options.output_file_name)
  else:
    output_file = os.path.join(
        options.output_directory, '%s.image.tar.gz' % digest)

  os.rename(temp_file_name, output_file)
  logging.info('Created tar.gz file at %s' % output_file)

  if options.bucket:
    bucket = options.bucket
    if bucket.startswith('gs://'):
      output_bucket = '%s/%s' % (
          bucket, os.path.basename(output_file))
    else:
      output_bucket = 'gs://%s/%s' % (
          bucket, os.path.basename(output_file))

    # /usr/local/bin not in redhat root PATH by default
    if '/usr/local/bin' not in os.environ['PATH']:
      os.environ['PATH'] += ':/usr/local/bin'

    # TODO: Consider using boto library directly.
    cmd = ['gsutil', 'cp', output_file, output_bucket]
    retcode = subprocess.call(cmd)
    if retcode != 0:
      logging.critical('Failed to copy image to bucket. '
                       'gsutil returned %d. To retry, run the command: %s',
                       retcode, ' '.join(cmd))

      return -1
    logging.info('Uploaded image to %s', output_bucket)

    # If we've uploaded, then we can remove the local file.
    os.remove(output_file)

  if options.cleanup:
    shutil.rmtree(scratch_dir)
<|endoftext|>


<commit_before># Copyright (C) 2017 OpenIO SAS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import partial
import collections
from six.moves.urllib.parse import parse_qs, quote_plus
from swift.common.swob import HTTPBadRequest
from swift.common.utils import config_true_value, split_path
from swift.common.wsgi import reiterate
from oio.common.autocontainer import ContainerBuilder

class AutoContainerBase(object):

    BYPASS_QS = "bypass-autocontainer"
    BYPASS_HEADER = "X-bypass-autocontainer"

    def __init__(self, app, acct,
                 strip_v1=False, account_first=False, swift3_compat=False,
                 stop_at_first_match=True):
        self.app = app
        self.account = acct
        self.bypass_header_key = ("HTTP_" +
                                  self.BYPASS_HEADER.upper().replace('-', '_'))
        self.con_builder = ContainerBuilder()
        self.account_first = account_first
        self.swift3_compat = swift3_compat
        self.strip_v1 = strip_v1
        if (not stop_at_first_match and
                not hasattr(self.con_builder, 'alternatives')):
            raise ValueError("Disabling 'stop_at_first_match' parameter "
                             "is not supported with openio-sds < 4.2")
        self.stop_at_first_match = stop_at_first_match

    def should_bypass(self, env):
        """Should we bypass this filter?"""
        header = env.get(self.bypass_header_key, "").lower()
        query = parse_qs(env.get('QUERY_STRING', "")).get(self.BYPASS_QS, [""])
        return config_true_value(header) or config_true_value(query[0])

    def _extract_path(self, path):
        account = self.account
        # Remove leading '/' to be consistent with split_path()
        obj = path[1:]
        container = None

        if self.strip_v1:
            version, tail = split_path('/' + obj, 1, 2, True)
            if version in ('v1', 'v1.0'):
                obj = tail

        if self.account_first:
            account, tail = split_path('/' + obj, 1, 2, True)
            obj = tail

        if obj is not None and self.swift3_compat:
            container, tail = split_path('/' + obj, 1, 2, True)
            obj = tail

        return account, container, obj

    def _convert_path(self, path):
        account, container, obj = self._extract_path(path)
        if obj is not None:
            container = quote_plus(self.con_builder(obj))
        return account, container, obj

    def _alternatives(self, account, container, obj):
        if obj is None:
            yield account, container, obj
        elif self.stop_at_first_match:
            yield account, quote_plus(self.con_builder(obj)), obj
        else:
            for alt_container in self.con_builder.alternatives(obj):
                yield account, quote_plus(alt_container), obj
        raise StopIteration

    @staticmethod
    def is_copy(env):
        """Tell if `env` represents an object copy operation."""
        return env['REQUEST_METHOD'] == 'PUT' and 'HTTP_X_COPY_FROM' in env

    @staticmethod
    def _save_response(env, status, headers, exc_info=None):
        env['last_status'] = status
        env['last_headers'] = headers
        env['last_exc_info'] = exc_info
        if 'first_status' not in env:
            env['first_status'] = status
            env['first_headers'] = headers
            env['first_exc_info'] = exc_info

    def _retry_loop(self, orig_env, start_response, path_to_modify,
                    env_modifier, alt_checker=None):
        """
        :param env_modifier: function copies and modifies the env
            dictionary, according to the alternative path parts
        :param alt_checker: function that checks the alternative
            path parts (may raise an exception)
        """
        local_env = {}
        query = parse_qs(orig_env['QUERY_STRING'], True)
        account, container, obj = self._extract_path(path_to_modify)
        is_container_req = container is not None and obj is None
        if is_container_req and 'prefix' in query:
            obj = query['prefix'][0]
        for alt in self._alternatives(account, container, obj):
            if alt_checker and not alt_checker(alt):
                return self.app(orig_env, start_response)
            env = env_modifier(orig_env, alt[:2] if is_container_req else alt)
            resp = self.app(env, partial(self._save_response, local_env))

            if isinstance(resp, collections.Iterable):
                # TODO when a MPU is completed, resp is an iterator
                # and we must fill last_status but it may break
                # next test
                resp = reiterate(resp)

            if 'last_status' not in local_env:
                # start_response() was not called. This happens when there is
                # no 'proxy-logging' just after 'catch_errors' in the pipeline.
                return resp
            if not local_env['last_status'].startswith('404'):
                # start_response() was called, and status is not 404.
                start_response(local_env['last_status'],
                               local_env['last_headers'],
                               local_env['last_exc_info'])
                return resp
            if 'first_resp' not in local_env:
                local_env['first_resp'] = resp

        start_response(local_env['first_status'],
                       local_env['first_headers'],
                       local_env['first_exc_info'])
        return local_env['first_resp']

    def _call_copy(self, env, start_response):
        """
        Run the retry loop (copy operations).
        """
        account, container, obj = self._convert_path(env.get('PATH_INFO'))
        if obj is None:
            # This is probably an account request
            return self.app(env, start_response)
        env['PATH_INFO'] = "/v1/%s/%s/%s" % (account, container, obj)

        # HTTP_X_COPY_FROM_ACCOUNT will just pass through
        if self.account_first:
            src_path = "/fake_account" + env['HTTP_X_COPY_FROM']
        else:
            src_path = env['HTTP_X_COPY_FROM']

        def modify_copy_from(orig_env, alternative):
            env_ = orig_env.copy()
            env_['HTTP_X_COPY_FROM'] = "/%s/%s" % (
                quote_plus(alternative[1]), alternative[2])
            return env_

        def check_container_obj(alternative):
            if not alternative[1] or not alternative[2]:
                raise HTTPBadRequest(body="Malformed copy-source header")
            return True

        return self._retry_loop(
            env, start_response, src_path,
            env_modifier=modify_copy_from,
            alt_checker=check_container_obj)

    def _call(self, env, start_response):
        """
        Run the retry loop (regular operations).
        """
        def modify_path_info(orig_env, alternative):
            env_ = orig_env.copy()
            env_['PATH_INFO'] = '/'.join(('', 'v1') + alternative)
            return env_

        def check_obj(alternative):
            return alternative[2] is not None

        return self._retry_loop(
            env, start_response, env['PATH_INFO'],
            env_modifier=modify_path_info,
            alt_checker=check_obj)

    def __call__(self, env, start_response):
        if self.should_bypass(env):
            return self.app(env, start_response)

        if self.is_copy(env):
            return self._call_copy(env, start_response)
        else:
            return self._call(env, start_response)
<commit_msg>Autocontainer: keep +segments suffix for MPU<commit_after># Copyright (C) 2017 OpenIO SAS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import partial
import collections
from six.moves.urllib.parse import parse_qs, quote_plus
from swift.common.swob import HTTPBadRequest
from swift.common.utils import config_true_value, split_path
from swift.common.wsgi import reiterate
from oio.common.autocontainer import ContainerBuilder

class AutoContainerBase(object):

    BYPASS_QS = "bypass-autocontainer"
    BYPASS_HEADER = "X-bypass-autocontainer"

    def __init__(self, app, acct,
                 strip_v1=False, account_first=False, swift3_compat=False,
                 stop_at_first_match=True):
        self.app = app
        self.account = acct
        self.bypass_header_key = ("HTTP_" +
                                  self.BYPASS_HEADER.upper().replace('-', '_'))
        self.con_builder = ContainerBuilder()
        self.account_first = account_first
        self.swift3_compat = swift3_compat
        self.strip_v1 = strip_v1
        if (not stop_at_first_match and
                not hasattr(self.con_builder, 'alternatives')):
            raise ValueError("Disabling 'stop_at_first_match' parameter "
                             "is not supported with openio-sds < 4.2")
        self.stop_at_first_match = stop_at_first_match

    def should_bypass(self, env):
        """Should we bypass this filter?"""
        header = env.get(self.bypass_header_key, "").lower()
        query = parse_qs(env.get('QUERY_STRING', "")).get(self.BYPASS_QS, [""])
        return config_true_value(header) or config_true_value(query[0])

    def _extract_path(self, path):
        account = self.account
        # Remove leading '/' to be consistent with split_path()
        obj = path[1:]
        container = None

        if self.strip_v1:
            version, tail = split_path('/' + obj, 1, 2, True)
            if version in ('v1', 'v1.0'):
                obj = tail

        if self.account_first:
            account, tail = split_path('/' + obj, 1, 2, True)
            obj = tail

        if obj is not None and self.swift3_compat:
            container, tail = split_path('/' + obj, 1, 2, True)
            obj = tail

        return account, container, obj

    def _convert_path(self, path):
        account, container, obj = self._extract_path(path)
        if obj is not None:
            container = quote_plus(self.con_builder(obj))
        return account, container, obj

    def _alternatives(self, account, container, obj):
        # put S3 parts in dedicated container
        suffix = ("+segments" if container and container.endswith('+segments')
                  else "")
        if obj is None:
            yield account, container, obj
        elif self.stop_at_first_match:
            yield account, quote_plus(self.con_builder(obj)) + suffix, obj
        else:
            for alt_container in self.con_builder.alternatives(obj):
                yield account, quote_plus(alt_container) + suffix, obj
        raise StopIteration

    @staticmethod
    def is_copy(env):
        """Tell if `env` represents an object copy operation."""
        return env['REQUEST_METHOD'] == 'PUT' and 'HTTP_X_COPY_FROM' in env

    @staticmethod
    def _save_response(env, status, headers, exc_info=None):
        env['last_status'] = status
        env['last_headers'] = headers
        env['last_exc_info'] = exc_info
        if 'first_status' not in env:
            env['first_status'] = status
            env['first_headers'] = headers
            env['first_exc_info'] = exc_info

    def _retry_loop(self, orig_env, start_response, path_to_modify,
                    env_modifier, alt_checker=None):
        """
        :param env_modifier: function copies and modifies the env
            dictionary, according to the alternative path parts
        :param alt_checker: function that checks the alternative
            path parts (may raise an exception)
        """
        local_env = {}
        query = parse_qs(orig_env['QUERY_STRING'], True)
        account, container, obj = self._extract_path(path_to_modify)
        is_container_req = container is not None and obj is None
        if is_container_req and 'prefix' in query:
            obj = query['prefix'][0]
        for alt in self._alternatives(account, container, obj):
            if alt_checker and not alt_checker(alt):
                return self.app(orig_env, start_response)
            env = env_modifier(orig_env, alt[:2] if is_container_req else alt)
            resp = self.app(env, partial(self._save_response, local_env))

            if isinstance(resp, collections.Iterable):
                # TODO when a MPU is completed, resp is an iterator
                # and we must fill last_status but it may break
                # next test
                resp = reiterate(resp)

            if 'last_status' not in local_env:
                # start_response() was not called. This happens when there is
                # no 'proxy-logging' just after 'catch_errors' in the pipeline.
                return resp
            if not local_env['last_status'].startswith('404'):
                # start_response() was called, and status is not 404.
                start_response(local_env['last_status'],
                               local_env['last_headers'],
                               local_env['last_exc_info'])
                return resp
            if 'first_resp' not in local_env:
                local_env['first_resp'] = resp

        start_response(local_env['first_status'],
                       local_env['first_headers'],
                       local_env['first_exc_info'])
        return local_env['first_resp']

    def _call_copy(self, env, start_response):
        """
        Run the retry loop (copy operations).
        """
        account, container, obj = self._convert_path(env.get('PATH_INFO'))
        if obj is None:
            # This is probably an account request
            return self.app(env, start_response)
        env['PATH_INFO'] = "/v1/%s/%s/%s" % (account, container, obj)

        # HTTP_X_COPY_FROM_ACCOUNT will just pass through
        if self.account_first:
            src_path = "/fake_account" + env['HTTP_X_COPY_FROM']
        else:
            src_path = env['HTTP_X_COPY_FROM']

        def modify_copy_from(orig_env, alternative):
            env_ = orig_env.copy()
            env_['HTTP_X_COPY_FROM'] = "/%s/%s" % (
                quote_plus(alternative[1]), alternative[2])
            return env_

        def check_container_obj(alternative):
            if not alternative[1] or not alternative[2]:
                raise HTTPBadRequest(body="Malformed copy-source header")
            return True

        return self._retry_loop(
            env, start_response, src_path,
            env_modifier=modify_copy_from,
            alt_checker=check_container_obj)

    def _call(self, env, start_response):
        """
        Run the retry loop (regular operations).
        """
        def modify_path_info(orig_env, alternative):
            env_ = orig_env.copy()
            env_['PATH_INFO'] = '/'.join(('', 'v1') + alternative)
            return env_

        def check_obj(alternative):
            return alternative[2] is not None

        return self._retry_loop(
            env, start_response, env['PATH_INFO'],
            env_modifier=modify_path_info,
            alt_checker=check_obj)

    def __call__(self, env, start_response):
        if self.should_bypass(env):
            return self.app(env, start_response)

        if self.is_copy(env):
            return self._call_copy(env, start_response)
        else:
            return self._call(env, start_response)
<|endoftext|>


<commit_before>from numpy.testing import assert_array_equal, run_module_suite
from skimage import data
from skimage.transform import (pyramid_reduce, pyramid_expand,
                               pyramid_gaussian, pyramid_laplacian)

image = data.lena()

def test_pyramid_reduce():
    rows, cols, dim = image.shape
    out = pyramid_reduce(image, downscale=2)
    assert_array_equal(out.shape, (rows / 2, cols / 2, dim))

def test_pyramid_expand():
    rows, cols, dim = image.shape
    out = pyramid_expand(image, upscale=2)
    assert_array_equal(out.shape, (rows * 2, cols * 2, dim))

def test_build_gaussian_pyramid():
    rows, cols, dim = image.shape
    pyramid = pyramid_gaussian(image, downscale=2)

    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer, dim)
        assert_array_equal(out.shape, layer_shape)

def test_build_laplacian_pyramid():
    rows, cols, dim = image.shape
    pyramid = pyramid_laplacian(image, downscale=2)

    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer, dim)
        assert_array_equal(out.shape, layer_shape)

if __name__ == "__main__":
    run_module_suite()
<commit_msg>Full test coverage for image pyramids<commit_after>from numpy.testing import assert_array_equal, assert_raises, run_module_suite
from skimage import data
from skimage.transform import pyramids

image = data.lena()
image_gray = image[..., 0]

def test_pyramid_reduce():
    # RGB image
    rows, cols, dim = image.shape
    out = pyramids.pyramid_reduce(image, downscale=2)
    assert_array_equal(out.shape, (rows / 2, cols / 2, dim))

    # grayscale image
    rows, cols = image_gray.shape
    out = pyramids.pyramid_reduce(image_gray, downscale=2)
    assert_array_equal(out.shape, (rows / 2, cols / 2))

def test_pyramid_expand():
    # RGB image
    rows, cols, dim = image.shape
    out = pyramids.pyramid_expand(image, upscale=2)
    assert_array_equal(out.shape, (rows * 2, cols * 2, dim))

    # grayscale image
    rows, cols = image_gray.shape
    out = pyramids.pyramid_expand(image_gray, upscale=2)
    assert_array_equal(out.shape, (rows * 2, cols * 2))

def test_build_gaussian_pyramid():
    # RGB image
    rows, cols, dim = image.shape
    pyramid = pyramids.pyramid_gaussian(image, downscale=2)
    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer, dim)
        assert_array_equal(out.shape, layer_shape)

    # grayscale image
    rows, cols = image_gray.shape
    pyramid = pyramids.pyramid_gaussian(image_gray, downscale=2)
    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer)
        assert_array_equal(out.shape, layer_shape)

def test_build_laplacian_pyramid():
    # RGB image
    rows, cols, dim = image.shape
    pyramid = pyramids.pyramid_laplacian(image, downscale=2)
    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer, dim)
        assert_array_equal(out.shape, layer_shape)

    # grayscale image
    rows, cols = image_gray.shape
    pyramid = pyramids.pyramid_laplacian(image_gray, downscale=2)
    for layer, out in enumerate(pyramid):
        layer_shape = (rows / 2 ** layer, cols / 2 ** layer)
        assert_array_equal(out.shape, layer_shape)

def test_check_factor():
    assert_raises(ValueError, pyramids._check_factor, 0.99)
    assert_raises(ValueError, pyramids._check_factor, - 2)

if __name__ == "__main__":
    run_module_suite()
<|endoftext|>


<commit_before>
""" File:           devices.py
    Author:         Andreas Poehlmann
    Last change:    2012/09/04

    Python Interface for OceanOptics Spectometers.
    Current device classes:
        * USB2000+
"""

#----------------------------------------------------------
import usb.core
import struct
from defines import OceanOpticsError as _OOError
import numpy as np
#----------------------------------------------------------

class USB2000(object):
    """ class USB2000:

            Serial --> serial_number
            acquire_spectrum() --> np.array
            device_temperature() --> float celcius
            integration_time(time_us=None) --> returns / sets the ~
    """

    def __init__(self):

        self._dev = usb.core.find(idVendor=0x2457, idProduct=0x101E)
        if self._dev is None:
            raise _OOError('No OceanOptics USB2000+ spectrometer found!')
        else:
            print ('*NOTE*: Currently the first device matching the '
                   'Vendor/Product id is used')

        # This information comes from the OEM-Datasheet
        # "http://www.oceanoptics.com/technical"
        #       "/engineering/OEM%20Data%20Sheet%20--%20USB2000+.pdf"
        self._EP1_out = 0x01
        self._EP1_in = 0x81
        self._EP2_in = 0x82
        self._EP6_in = 0x86
        self._EP1_in_size = 64
        self._EP2_in_size = 512
        self._EP6_in_size = 512

        # This part makes the initialization a little bit more robust
        self._dev.set_configuration()
        self._initialize()
        #<robust>#
        while True:
            try:
                self._usbcomm = self._query_status()['usb_speed']
                break
            except usb.core.USBError:
                pass
        while True:
            try:
                self._request_spectrum()
                break
            except:
                pass
        #</robust>#

        self._wl = self._get_wavelength_calibration()
        self._nl = self._get_nonlinearity_calibration()
        self._st = self._get_saturation_calibration()
        self.Serial = self._get_serial()

    def integration_time(self, time_us=None):
        if not (time_us is None):
            self._set_integration_time(time_us)
        return self._query_status()['integration_time']

    def device_temperature(self):
        return self._read_pcb_temperature()

    def acquire_spectrum(self):
        raw_intensity = np.array(self._request_spectrum())[20:] # self.st
        wavelength = sum( self._wl[i] * np.arange(20,2048)**i for i in range(4) )
        intensity = sum( self._nl[i] * raw_intensity**i for i in range(8) )
        return np.vstack([wavelength, intensity])

    #-----------------------------
    # The user doesn't need to see this
    #-----------------------------
    def _initialize(self):
        """ 0x01 initialize """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x01))

    def _set_integration_time(self, time_us):
        """ 0x02 set integration time """
        self._dev.write(self._EP1_out, struct.pack('<BI', 0x02, int(time_us)))

    def _query_information(self, address, raw=False):
        """ 0x05 query info """
        self._dev.write(self._EP1_out, struct.pack('<BB', 0x05, int(address)%0xFF))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if bool(raw): return ret
        if ret[0] != 0x05 or ret[1] != int(address)%0xFF:
            raise Exception('query_information: Wrong answer')
        return ret[2:ret[2:].index(0)+2].tostring()

    def _read_register_information(self, address):
        """ 0x6B read register info """
        self._dev.write(self._EP1_out, struct.pack('<BB', 0x6B, int(address)%0xFF))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if ret[0] != int(address)%0xFF:
            raise Exception('read_register_information: Wrong answer')
        return struct.unpack('<h', ret[1:])[0]

    def _read_pcb_temperature(self):
        """ 0x6C read pcb temperature """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x6C))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if ret[0] != 0x08:
            raise Exception('read_pcb_temperature: Wrong answer')
        adc, = struct.unpack('<h', ret[1:])
        return 0.003906*adc

    def _query_status(self):
        """ 0xFE query status """
        self._dev.write(self._EP1_out, struct.pack('<B', 0xFE))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        data = struct.unpack('<HLBBBBBBBBBB', ret[:])
        ret = { 'pixels'              : data[0],
                'integration_time'    : data[1],
                'lamp_enable'         : data[2],
                'trigger_mode'        : data[3],
                'acquisition_status'  : data[4],
                'packets_in_spectrum' : data[5],
                'power_down'          : data[6],
                'packets_in_endpoint' : data[7],
                'usb_speed'           : data[10] }
        return ret

    def _request_spectrum(self):
        """ 0x09 request spectra """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x09))
        if self._usbcomm == 0x80: #HIGHSPEED
            ret = [self._dev.read(self._EP2_in, 512) for _ in range(8)]
        else: # _usbcomm == 0x00  #FULLSPEED
            ret = [self._dev.read(self._EP2_in, 64) for _ in range(64)]
        ret = sum(ret[1:], ret[0])
        spectrum = struct.unpack('<'+'h'*2048, ret)
        sync = self._dev.read(self._EP2_in, 1)
        if sync[0] != 0x69:
            raise _OOError('request_spectrum: Wrong answer')
        return spectrum

    def _get_serial(self):
        return self._query_information(0)

    def _get_wavelength_calibration(self):
        return [float(self._query_information(i)) for i in range(1,5)]

    def _get_nonlinearity_calibration(self):
        if int(self._query_information(14)) != 7:
            # Don't care about this right now
            raise _OOError('This spectrometer has less correction factors')
        return [float(self._query_information(i)) for i in range(6,14)]

    def _get_saturation_calibration(self):
        ret = self._query_information(0x11, raw=True)
        return 65535.0/float(struct.unpack('<h',ret[6:8])[0])

    #-------------------------------------------
    # This stuff is not implemented yet.
    # Don't really need it ...
    #-------------------------------------------
    def _set_strobe_enable_status(self):
        """ 0x03 set strobe enable status """
        raise NotImplementedError

    def _set_shutdown_mode(self):
        """ 0x04 set shutdown mode """
        raise NotImplementedError

    def _write_information(self, address):
        """ 0x06 write info """
        raise NotImplementedError

    def _set_trigger_mode(self, mode):
        """ 0x0A set trigger mode """
        raise NotImplementedError

    def _query_plugin_num(self):
        """ 0x0B query number of plugin accessories """
        raise NotImplementedError

    def _query_plugin_ident(self):
        """ 0x0C query plugin identifiers """
        raise NotImplementedError

    def _detect_plugins(self):
        """ 0x0D detect plugins """
        raise NotImplementedError

    def _i2c_read(self):
        """ 0x60 I2C read """
        raise NotImplementedError

    def _i2c_write(self, data):
        """ 0x61 I2C write """
        raise NotImplementedError

    def _spi_io(self):
        """ 0x62 spi io """
        raise NotImplementedError

    def _write_register_info(self):
        """ 0x6A write register info """
        raise NotImplementedError

    def _read_irradiance_calibration(self):
        """ 0x6D read irradiance calib factors """
        raise NotImplementedError

if __name__ == '__main__':

    from gi.repository import Gtk, GLib

    class mpl:
        from matplotlib.figure import Figure
        from matplotlib.backends.backend_gtk3agg import FigureCanvasGTK3Agg as FigureCanvas

    class DynamicPlotter(Gtk.Window):

        def __init__(self, sampleinterval=0.1, size=(600,350), raw=False):
            # Gtk stuff
            Gtk.Window.__init__(self, title='OceanOptics USB2000+ Spectrum')
            self.connect("destroy", lambda x : Gtk.main_quit())
            self.set_default_size(*size)
            # Data stuff
            self._interval = int(sampleinterval*1000)
            self.spec = USB2000()
            self.wl, self.sp = self.spec.acquire_spectrum()
            self.raw = bool(raw)
            # MPL stuff
            self.figure = mpl.Figure()
            self.ax = self.figure.add_subplot(1, 1, 1)
            self.ax.grid(True)
            self.canvas = mpl.FigureCanvas(self.figure)
            self.line, = self.ax.plot(self.wl, self.sp)
            # Gtk stuff
            self.add(self.canvas)
            self.canvas.show()
            self.show_all()

        def updateplot(self):
            if self.raw:
                self.sp = np.array(self.spec._request_spectrum())[20:]
            else:
                _, self.sp = self.spec.acquire_spectrum()
            self.line.set_ydata(self.sp)
            self.ax.relim()
            self.ax.autoscale_view(False, False, True)
            self.canvas.draw()
            return True

        def run(self):
            GLib.timeout_add(self._interval, self.updateplot )
            Gtk.main()

    m = DynamicPlotter(sampleinterval=0.05)
    m.run()

<commit_msg>fixed saturation error<commit_after>
""" File:           devices.py
    Author:         Andreas Poehlmann
    Last change:    2012/09/04

    Python Interface for OceanOptics Spectometers.
    Current device classes:
        * USB2000+
"""

#----------------------------------------------------------
import usb.core
import struct
from defines import OceanOpticsError as _OOError
import numpy as np
#----------------------------------------------------------

class USB2000(object):
    """ class USB2000:

            Serial --> serial_number
            acquire_spectrum() --> np.array
            device_temperature() --> float celcius
            integration_time(time_us=None) --> returns / sets the ~
    """

    def __init__(self):

        self._dev = usb.core.find(idVendor=0x2457, idProduct=0x101E)
        if self._dev is None:
            raise _OOError('No OceanOptics USB2000+ spectrometer found!')
        else:
            print ('*NOTE*: Currently the first device matching the '
                   'Vendor/Product id is used')

        # This information comes from the OEM-Datasheet
        # "http://www.oceanoptics.com/technical"
        #       "/engineering/OEM%20Data%20Sheet%20--%20USB2000+.pdf"
        self._EP1_out = 0x01
        self._EP1_in = 0x81
        self._EP2_in = 0x82
        self._EP6_in = 0x86
        self._EP1_in_size = 64
        self._EP2_in_size = 512
        self._EP6_in_size = 512

        # This part makes the initialization a little bit more robust
        self._dev.set_configuration()
        self._initialize()
        #<robust>#
        while True:
            try:
                self._usbcomm = self._query_status()['usb_speed']
                break
            except usb.core.USBError:
                pass
        while True:
            try:
                self._request_spectrum()
                break
            except:
                pass
        #</robust>#

        self._wl = self._get_wavelength_calibration()
        self._nl = self._get_nonlinearity_calibration()
        self._st = self._get_saturation_calibration()
        self.Serial = self._get_serial()

    def integration_time(self, time_us=None):
        if not (time_us is None):
            self._set_integration_time(time_us)
        return self._query_status()['integration_time']

    def device_temperature(self):
        return self._read_pcb_temperature()

    def acquire_spectrum(self):
        raw_intensity = np.array(self._request_spectrum(), dtype=np.float)[20:] # self.st
        wavelength = sum( self._wl[i] * np.arange(20,2048)**i for i in range(4) )
        intensity =  sum( self._nl[i] * raw_intensity**i for i in range(8) )
        return np.vstack([wavelength, intensity])

    #-----------------------------
    # The user doesn't need to see this
    #-----------------------------
    def _initialize(self):
        """ 0x01 initialize """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x01))

    def _set_integration_time(self, time_us):
        """ 0x02 set integration time """
        self._dev.write(self._EP1_out, struct.pack('<BI', 0x02, int(time_us)))

    def _query_information(self, address, raw=False):
        """ 0x05 query info """
        self._dev.write(self._EP1_out, struct.pack('<BB', 0x05, int(address)%0xFF))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if bool(raw): return ret
        if ret[0] != 0x05 or ret[1] != int(address)%0xFF:
            raise Exception('query_information: Wrong answer')
        return ret[2:ret[2:].index(0)+2].tostring()

    def _read_register_information(self, address):
        """ 0x6B read register info """
        self._dev.write(self._EP1_out, struct.pack('<BB', 0x6B, int(address)%0xFF))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if ret[0] != int(address)%0xFF:
            raise Exception('read_register_information: Wrong answer')
        return struct.unpack('<h', ret[1:])[0]

    def _read_pcb_temperature(self):
        """ 0x6C read pcb temperature """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x6C))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        if ret[0] != 0x08:
            raise Exception('read_pcb_temperature: Wrong answer')
        adc, = struct.unpack('<h', ret[1:])
        return 0.003906*adc

    def _query_status(self):
        """ 0xFE query status """
        self._dev.write(self._EP1_out, struct.pack('<B', 0xFE))
        ret = self._dev.read(self._EP1_in, self._EP1_in_size)
        data = struct.unpack('<HLBBBBBBBBBB', ret[:])
        ret = { 'pixels'              : data[0],
                'integration_time'    : data[1],
                'lamp_enable'         : data[2],
                'trigger_mode'        : data[3],
                'acquisition_status'  : data[4],
                'packets_in_spectrum' : data[5],
                'power_down'          : data[6],
                'packets_in_endpoint' : data[7],
                'usb_speed'           : data[10] }
        return ret

    def _request_spectrum(self):
        """ 0x09 request spectra """
        self._dev.write(self._EP1_out, struct.pack('<B', 0x09))
        if self._usbcomm == 0x80: #HIGHSPEED
            ret = [self._dev.read(self._EP2_in, 512) for _ in range(8)]
        else: # _usbcomm == 0x00  #FULLSPEED
            ret = [self._dev.read(self._EP2_in, 64) for _ in range(64)]
        ret = sum(ret[1:], ret[0])
        spectrum = struct.unpack('<'+'h'*2048, ret)
        sync = self._dev.read(self._EP2_in, 1)
        if sync[0] != 0x69:
            raise _OOError('request_spectrum: Wrong answer')
        return spectrum

    def _get_serial(self):
        return self._query_information(0)

    def _get_wavelength_calibration(self):
        return [float(self._query_information(i)) for i in range(1,5)]

    def _get_nonlinearity_calibration(self):
        if int(self._query_information(14)) != 7:
            # Don't care about this right now
            raise _OOError('This spectrometer has less correction factors')
        return [float(self._query_information(i)) for i in range(6,14)]

    def _get_saturation_calibration(self):
        ret = self._query_information(0x11, raw=True)
        return 65535.0/float(struct.unpack('<h',ret[6:8])[0])

    #-------------------------------------------
    # This stuff is not implemented yet.
    # Don't really need it ...
    #-------------------------------------------
    def _set_strobe_enable_status(self):
        """ 0x03 set strobe enable status """
        raise NotImplementedError

    def _set_shutdown_mode(self):
        """ 0x04 set shutdown mode """
        raise NotImplementedError

    def _write_information(self, address):
        """ 0x06 write info """
        raise NotImplementedError

    def _set_trigger_mode(self, mode):
        """ 0x0A set trigger mode """
        raise NotImplementedError

    def _query_plugin_num(self):
        """ 0x0B query number of plugin accessories """
        raise NotImplementedError

    def _query_plugin_ident(self):
        """ 0x0C query plugin identifiers """
        raise NotImplementedError

    def _detect_plugins(self):
        """ 0x0D detect plugins """
        raise NotImplementedError

    def _i2c_read(self):
        """ 0x60 I2C read """
        raise NotImplementedError

    def _i2c_write(self, data):
        """ 0x61 I2C write """
        raise NotImplementedError

    def _spi_io(self):
        """ 0x62 spi io """
        raise NotImplementedError

    def _write_register_info(self):
        """ 0x6A write register info """
        raise NotImplementedError

    def _read_irradiance_calibration(self):
        """ 0x6D read irradiance calib factors """
        raise NotImplementedError

if __name__ == '__main__':

    from gi.repository import Gtk, GLib

    class mpl:
        from matplotlib.figure import Figure
        from matplotlib.backends.backend_gtk3agg import FigureCanvasGTK3Agg as FigureCanvas

    class DynamicPlotter(Gtk.Window):

        def __init__(self, sampleinterval=0.1, size=(600,350), raw=False):
            # Gtk stuff
            Gtk.Window.__init__(self, title='OceanOptics USB2000+ Spectrum')
            self.connect("destroy", lambda x : Gtk.main_quit())
            self.set_default_size(*size)
            # Data stuff
            self._interval = int(sampleinterval*1000)
            self.spec = USB2000()
            self.wl, self.sp = self.spec.acquire_spectrum()
            self.raw = bool(raw)
            # MPL stuff
            self.figure = mpl.Figure()
            self.ax = self.figure.add_subplot(1, 1, 1)
            self.ax.grid(True)
            self.canvas = mpl.FigureCanvas(self.figure)
            self.line, = self.ax.plot(self.wl, self.sp)
            # Gtk stuff
            self.add(self.canvas)
            self.canvas.show()
            self.show_all()

        def updateplot(self):
            if self.raw:
                self.sp = np.array(self.spec._request_spectrum())[20:]
            else:
                _, self.sp = self.spec.acquire_spectrum()
            self.line.set_ydata(self.sp)
            self.ax.relim()
            self.ax.autoscale_view(False, False, True)
            self.canvas.draw()
            return True

        def run(self):
            GLib.timeout_add(self._interval, self.updateplot )
            Gtk.main()

    import sys
    if sys.argv[1:] == ['--raw']:
        raw=True
    else:
        raw=False

    m = DynamicPlotter(sampleinterval=0.05, raw=raw)
    m.run()

<|endoftext|>


<commit_before>#!/usr/bin/env python

import atexit
import serial
import os
import logging
import logging.handlers
import sys
import time
import subprocess
import sh
import signal
import re
import struct

from datetime import datetime, timedelta

logger = logging.getLogger('dreampi')

def autoconfigure_ppp(device, speed):
    """
       Every network is different, this function runs on boot and tries
       to autoconfigure PPP as best it can by detecting the subnet and gateway
       we're running on
    """

    gateway_ip = subprocess.check_output("route -n | grep 'UG[ \t]' | awk '{print $2}'", shell=True)
    subnet = gateway_ip.split(".")[:3]

    def find_unused_ips():
        try:
            ARP_SCAN_COMMAND = [ "sudo", "arp-scan", "{}.{}.{}.0/24".format(*subnet) ]
            scan_results = subprocess.check_output(ARP_SCAN_COMMAND)
            used_ips = set([int(x.split()[0].split(".")[-1]) for x in scan_results.split("\n")[2:-4]])
            free_ips = ("{}.{}.{}.{}".format(*(subnet + [x])) for x in range(99, 1, -1) if x not in used_ips)
            return free_ips.next(), free_ips.next()
        except:
            logger.exception("Couldn't detect free IPs. Using .98 and .99")
            return "{}.{}.{}.98".format(*subnet), "{}.{}.{}.99".format(*subnet)

    PEERS_TEMPLATE = """
{device}
{device_speed}
{this_ip}:{dc_ip}
noauth
    """.strip()

    OPTIONS_TEMPLATE = """
debug
require-pap
ms-dns {}
proxyarp
ktune
    """.strip()

    this_ip, dreamcast_ip = find_unused_ips()

    peers_content = PEERS_TEMPLATE.format(device=device, device_speed=speed, this_ip=this_ip, dc_ip=dreamcast_ip)

    with open("/etc/ppp/peers/dreamcast", "w") as f:
        f.write(peers_content)

    options_content = OPTIONS_TEMPLATE.format(this_ip)

    with open("/etc/ppp/options", "w") as f:
        f.write(options_content)

def detect_device_and_speed():
    command = [ "wvdialconf", "/dev/null" ]

    try:
        output = subprocess.check_output(command, stderr=subprocess.STDOUT)

        lines = output.split("\n")

        for line in lines:
            match = re.match("(.+)\<Info\>\:\sSpeed\s(\d+);", line.strip())
            if match:
                device = match.group(1)
                speed = match.group(2)
                logger.info("Detected device {} with speed {}".format(device, speed))
                return device, speed
        else:
            logger.info("No device detected")

    except OSError:
        logger.warning("Unable to detect modem. Falling back to ttyACM0")
    return ("ttyACM0", 460800)

class Daemon(object):
    def __init__(self, pidfile, process):
        self.pidfile = pidfile
        self.process = process

    def daemonize(self):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)

        except OSError, e:
            sys.exit(1)

        os.chdir("/")
        os.setsid()
        os.umask(0)

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.exit(1)

        atexit.register(self.delete_pid)
        pid = str(os.getpid())
        with open(self.pidfile, 'w+') as f:
            f.write("%s\n" % pid)

    def delete_pid(self):
        os.remove(self.pidfile)

    def _read_pid_from_pidfile(self):
        try:
            with open(self.pidfile, 'r') as pf:
                pid = int(pf.read().strip())
        except IOError:
            pid = None
        return pid

    def start(self):
        pid = self._read_pid_from_pidfile()

        if pid:
            logger.info("Daemon already running, exiting")
            sys.exit(1)

        logger.info("Starting daemon")
        self.daemonize()
        self.run()

    def stop(self):
        pid = self._read_pid_from_pidfile()

        if not pid:
            logger.info("pidfile doesn't exist, deamon must not be running")
            return

        try:
            while True:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)

        except OSError as err:
            if os.path.exists(self.pidfile):
                os.remove(self.pidfile)
            else:
                sys.exit(1)

    def restart(self):
        self.stop()
        self.start()

    def run(self):
        self.process()

class Modem(object):
    def __init__(self, comm_speed=None, send_dial_tone=True):
        self._device, self._speed = detect_device_and_speed()
        self._comm_speed = comm_speed or self._speed
        self._serial = None
        self._sending_tone = False

        if send_dial_tone:
            self._dial_tone_wav = self._read_dial_tone()
        else:
            self._dial_tone_wav = None

        self._time_since_last_dial_tone = None
        self._dial_tone_counter = 0

    @property
    def device_speed(self):
        return self._speed

    @property
    def device_name(self):
        return self._device

    def _read_dial_tone(self):
        this_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
        dial_tone_wav = os.path.join(this_dir, "dial-tone.wav")

        with open(dial_tone_wav, "rb") as f:
            dial_tone = f.read() # Read the entire wav file
            dial_tone = dial_tone[44:] # Strip the header (44 bytes)

        return dial_tone

    def connect(self):
        if self._serial:
            self.disconnect()

        logger.info("Opening serial interface to {}".format(self._device))
        self._serial = serial.Serial("/dev/{}".format(self._device), self._comm_speed, timeout=0)

    def disconnect(self):
        if self._serial and self._serial.isOpen():
            self._serial.close()
            self._serial = None
            logger.info("Serial interface terminated")

    def reset(self):
        self.send_command("ATZ0") # Send reset command

    def start_dial_tone(self):
        if not self._dial_tone_wav:
            return

        self.reset()
        self.send_command("AT+FCLASS=8") # Enter voice mode
        self.send_command("AT+VLS=1") # Go off-hook
        self.send_command("AT+VSM=1,8000") # 8 bit unsigned PCM
        self.send_command("AT+VTX") # Voice transmission mode

        self._sending_tone = True
        self._time_since_last_dial_tone = datetime.now() - timedelta(seconds=100)
        self._dial_tone_counter = 0

    def stop_dial_tone(self):
        if not self._sending_tone:
            return

        self._serial.write("\0{}{}\r\n".format(chr(0x10), chr(0x03)))
        self.send_escape()
        self.send_command("ATH0") # Go on-hook
        self.reset() #Reset the modem
        self._sending_tone = False

    def answer(self):
        self.reset()
        self.send_command("ATA")
        time.sleep(2)
        logger.info("Call answered!")
        logger.info(subprocess.check_output(["pon", "dreamcast"]))
        logger.info("Connected")

    def send_command(self, command):
        final_command = "%s\r\n" % command
        self._serial.write(final_command)
        logger.info(final_command)

        line = self._serial.readline()
        while True:
            if "OK" in line or "ERROR" in line or "CONNECT" in line or "VCON" in line:
                logger.info(line)
                break

            line = self._serial.readline()

    def send_escape(self):
        time.sleep(1.0)
        self._serial.write("+++")
        time.sleep(1.0)

    def update(self):
        now = datetime.now()
        if self._sending_tone:
            # Keep sending dial tone
            BUFFER_LENGTH = 1000
            TIME_BETWEEN_UPLOADS_MS = (1000.0 / 8000.0) * BUFFER_LENGTH

            milliseconds = (now - self._time_since_last_dial_tone).microseconds * 1000
            if not self._time_since_last_dial_tone or milliseconds >= TIME_BETWEEN_UPLOADS_MS:
                byte = self._dial_tone_wav[self._dial_tone_counter:self._dial_tone_counter+BUFFER_LENGTH]
                self._dial_tone_counter += BUFFER_LENGTH
                if self._dial_tone_counter >= len(self._dial_tone_wav):
                    self._dial_tone_counter = 0
                self._serial.write(byte)
                self._time_since_last_dial_tone = now

def process():
    dial_tone_enabled = not "--disable-dial-tone" in sys.argv

    # Make sure pppd isn't running
    with open(os.devnull, 'wb') as devnull:
        subprocess.call(["sudo", "killall", "pppd"], stderr=devnull)

    modem = Modem(115200, dial_tone_enabled)
    autoconfigure_ppp(modem.device_name, modem.device_speed)

    mode = "LISTENING"

    modem.connect()
    if dial_tone_enabled:
        modem.start_dial_tone()

    time_digit_heard = None

    while True:
        now = datetime.now()

        if mode == "LISTENING":
            modem.update()
            char = modem._serial.read(1).strip()
            if not char:
                continue

            if ord(char) == 16:
                #DLE character
                try:
                    char = modem._serial.read(1)
                    digit = int(char)
                    logger.info("Heard: %s", digit)

                    mode = "ANSWERING"
                    modem.stop_dial_tone()
                    time_digit_heard = now
                except (TypeError, ValueError):
                    pass
        elif mode == "ANSWERING":
            if (now - time_digit_heard).total_seconds() > 8.0:
                time_digit_heard = None
                modem.answer()
                mode = "CONNECTED"

        elif mode == "CONNECTED":
            # We start watching /var/log/messages for the hang up message
            for line in sh.tail("-f", "/var/log/messages", "-n", "1", _iter=True):
                if "Modem hangup" in line:
                    logger.info("Detected modem hang up, going back to listening")
                    time.sleep(5) # Give the hangup some time
                    break

            mode = "LISTENING"
            modem.disconnect()
            modem = Modem(115200, dial_tone_enabled)
            modem.connect()
            if dial_tone_enabled:
                modem.start_dial_tone()

    return 0

def main():
    try:
        return process()
    except:
        logger.exception("Something went wrong...")
        return 1

if __name__ == '__main__':
    logger.setLevel(logging.INFO)
    handler = logging.handlers.SysLogHandler(address='/dev/log')
    logger.addHandler(handler)

    if len(sys.argv) > 1 and "--no-daemon" in sys.argv:
        logger.addHandler(logging.StreamHandler())
        sys.exit(main())

    daemon = Daemon("/tmp/dreampi.pid", main)

    if len(sys.argv) == 2:
        if sys.argv[1] == "start":
            daemon.start()
        elif sys.argv[1] == "stop":
            daemon.stop()
        elif sys.argv[1] == "restart":
            daemon.restart()
        else:
            sys.exit(2)
        sys.exit(0)
    else:
        print("Usage: %s start|stop|restart" % sys.argv[0])
        sys.exit(2)
<commit_msg>Don't try to negotiate compression (Starlancer gets it wrong) also, speed up the communication with the modem to full speed<commit_after>#!/usr/bin/env python

import atexit
import serial
import os
import logging
import logging.handlers
import sys
import time
import subprocess
import sh
import signal
import re
import struct

from datetime import datetime, timedelta

logger = logging.getLogger('dreampi')

def autoconfigure_ppp(device, speed):
    """
       Every network is different, this function runs on boot and tries
       to autoconfigure PPP as best it can by detecting the subnet and gateway
       we're running on
    """

    gateway_ip = subprocess.check_output("route -n | grep 'UG[ \t]' | awk '{print $2}'", shell=True)
    subnet = gateway_ip.split(".")[:3]

    def find_unused_ips():
        try:
            ARP_SCAN_COMMAND = [ "sudo", "arp-scan", "{}.{}.{}.0/24".format(*subnet) ]
            scan_results = subprocess.check_output(ARP_SCAN_COMMAND)
            used_ips = set([int(x.split()[0].split(".")[-1]) for x in scan_results.split("\n")[2:-4]])
            free_ips = ("{}.{}.{}.{}".format(*(subnet + [x])) for x in range(99, 1, -1) if x not in used_ips)
            return free_ips.next(), free_ips.next()
        except:
            logger.exception("Couldn't detect free IPs. Using .98 and .99")
            return "{}.{}.{}.98".format(*subnet), "{}.{}.{}.99".format(*subnet)

    PEERS_TEMPLATE = """
{device}
{device_speed}
{this_ip}:{dc_ip}
noauth
    """.strip()

    OPTIONS_TEMPLATE = """
debug
ms-dns {}
proxyarp
ktune
noccp
    """.strip()

    this_ip, dreamcast_ip = find_unused_ips()

    peers_content = PEERS_TEMPLATE.format(device=device, device_speed=speed, this_ip=this_ip, dc_ip=dreamcast_ip)

    with open("/etc/ppp/peers/dreamcast", "w") as f:
        f.write(peers_content)

    options_content = OPTIONS_TEMPLATE.format(this_ip)

    with open("/etc/ppp/options", "w") as f:
        f.write(options_content)

def detect_device_and_speed():
    command = [ "wvdialconf", "/dev/null" ]

    try:
        output = subprocess.check_output(command, stderr=subprocess.STDOUT)

        lines = output.split("\n")

        for line in lines:
            match = re.match("(.+)\<Info\>\:\sSpeed\s(\d+);", line.strip())
            if match:
                device = match.group(1)
                speed = match.group(2)
                logger.info("Detected device {} with speed {}".format(device, speed))
                return device, speed
        else:
            logger.info("No device detected")

    except OSError:
        logger.warning("Unable to detect modem. Falling back to ttyACM0")
    return ("ttyACM0", 460800)

class Daemon(object):
    def __init__(self, pidfile, process):
        self.pidfile = pidfile
        self.process = process

    def daemonize(self):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)

        except OSError, e:
            sys.exit(1)

        os.chdir("/")
        os.setsid()
        os.umask(0)

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.exit(1)

        atexit.register(self.delete_pid)
        pid = str(os.getpid())
        with open(self.pidfile, 'w+') as f:
            f.write("%s\n" % pid)

    def delete_pid(self):
        os.remove(self.pidfile)

    def _read_pid_from_pidfile(self):
        try:
            with open(self.pidfile, 'r') as pf:
                pid = int(pf.read().strip())
        except IOError:
            pid = None
        return pid

    def start(self):
        pid = self._read_pid_from_pidfile()

        if pid:
            logger.info("Daemon already running, exiting")
            sys.exit(1)

        logger.info("Starting daemon")
        self.daemonize()
        self.run()

    def stop(self):
        pid = self._read_pid_from_pidfile()

        if not pid:
            logger.info("pidfile doesn't exist, deamon must not be running")
            return

        try:
            while True:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)

        except OSError as err:
            if os.path.exists(self.pidfile):
                os.remove(self.pidfile)
            else:
                sys.exit(1)

    def restart(self):
        self.stop()
        self.start()

    def run(self):
        self.process()

class Modem(object):
    def __init__(self, comm_speed=None, send_dial_tone=True):
        self._device, self._speed = detect_device_and_speed()
        self._comm_speed = comm_speed or self._speed
        self._serial = None
        self._sending_tone = False

        if send_dial_tone:
            self._dial_tone_wav = self._read_dial_tone()
        else:
            self._dial_tone_wav = None

        self._time_since_last_dial_tone = None
        self._dial_tone_counter = 0

    @property
    def device_speed(self):
        return self._speed

    @property
    def device_name(self):
        return self._device

    def _read_dial_tone(self):
        this_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
        dial_tone_wav = os.path.join(this_dir, "dial-tone.wav")

        with open(dial_tone_wav, "rb") as f:
            dial_tone = f.read() # Read the entire wav file
            dial_tone = dial_tone[44:] # Strip the header (44 bytes)

        return dial_tone

    def connect(self):
        if self._serial:
            self.disconnect()

        logger.info("Opening serial interface to {}".format(self._device))
        self._serial = serial.Serial("/dev/{}".format(self._device), self._comm_speed, timeout=0)

    def disconnect(self):
        if self._serial and self._serial.isOpen():
            self._serial.close()
            self._serial = None
            logger.info("Serial interface terminated")

    def reset(self):
        self.send_command("ATZ0") # Send reset command

    def start_dial_tone(self):
        if not self._dial_tone_wav:
            return

        self.reset()
        self.send_command("AT+FCLASS=8") # Enter voice mode
        self.send_command("AT+VLS=1") # Go off-hook
        self.send_command("AT+VSM=1,8000") # 8 bit unsigned PCM
        self.send_command("AT+VTX") # Voice transmission mode

        self._sending_tone = True
        self._time_since_last_dial_tone = datetime.now() - timedelta(seconds=100)
        self._dial_tone_counter = 0

    def stop_dial_tone(self):
        if not self._sending_tone:
            return

        self._serial.write("\0{}{}\r\n".format(chr(0x10), chr(0x03)))
        self.send_escape()
        self.send_command("ATH0") # Go on-hook
        self.reset() #Reset the modem
        self._sending_tone = False

    def answer(self):
        self.reset()
        self.send_command("ATA")
        time.sleep(2)
        logger.info("Call answered!")
        logger.info(subprocess.check_output(["pon", "dreamcast"]))
        logger.info("Connected")

    def send_command(self, command):
        final_command = "%s\r\n" % command
        self._serial.write(final_command)
        logger.info(final_command)

        line = self._serial.readline()
        while True:
            if "OK" in line or "ERROR" in line or "CONNECT" in line or "VCON" in line:
                logger.info(line)
                break

            line = self._serial.readline()

    def send_escape(self):
        time.sleep(1.0)
        self._serial.write("+++")
        time.sleep(1.0)

    def update(self):
        now = datetime.now()
        if self._sending_tone:
            # Keep sending dial tone
            BUFFER_LENGTH = 1000
            TIME_BETWEEN_UPLOADS_MS = (1000.0 / 8000.0) * BUFFER_LENGTH

            milliseconds = (now - self._time_since_last_dial_tone).microseconds * 1000
            if not self._time_since_last_dial_tone or milliseconds >= TIME_BETWEEN_UPLOADS_MS:
                byte = self._dial_tone_wav[self._dial_tone_counter:self._dial_tone_counter+BUFFER_LENGTH]
                self._dial_tone_counter += BUFFER_LENGTH
                if self._dial_tone_counter >= len(self._dial_tone_wav):
                    self._dial_tone_counter = 0
                self._serial.write(byte)
                self._time_since_last_dial_tone = now

def process():
    dial_tone_enabled = not "--disable-dial-tone" in sys.argv

    # Make sure pppd isn't running
    with open(os.devnull, 'wb') as devnull:
        subprocess.call(["sudo", "killall", "pppd"], stderr=devnull)

    modem = Modem(None, dial_tone_enabled)
    autoconfigure_ppp(modem.device_name, modem.device_speed)

    mode = "LISTENING"

    modem.connect()
    if dial_tone_enabled:
        modem.start_dial_tone()

    time_digit_heard = None

    while True:
        now = datetime.now()

        if mode == "LISTENING":
            modem.update()
            char = modem._serial.read(1).strip()
            if not char:
                continue

            if ord(char) == 16:
                #DLE character
                try:
                    char = modem._serial.read(1)
                    digit = int(char)
                    logger.info("Heard: %s", digit)

                    mode = "ANSWERING"
                    modem.stop_dial_tone()
                    time_digit_heard = now
                except (TypeError, ValueError):
                    pass
        elif mode == "ANSWERING":
            if (now - time_digit_heard).total_seconds() > 8.0:
                time_digit_heard = None
                modem.answer()
                mode = "CONNECTED"

        elif mode == "CONNECTED":
            # We start watching /var/log/messages for the hang up message
            for line in sh.tail("-f", "/var/log/messages", "-n", "1", _iter=True):
                if "Modem hangup" in line:
                    logger.info("Detected modem hang up, going back to listening")
                    time.sleep(5) # Give the hangup some time
                    break

            mode = "LISTENING"
            modem.disconnect()
            modem = Modem(None, dial_tone_enabled)
            modem.connect()
            if dial_tone_enabled:
                modem.start_dial_tone()

    return 0

def main():
    try:
        return process()
    except:
        logger.exception("Something went wrong...")
        return 1

if __name__ == '__main__':
    logger.setLevel(logging.INFO)
    handler = logging.handlers.SysLogHandler(address='/dev/log')
    logger.addHandler(handler)

    if len(sys.argv) > 1 and "--no-daemon" in sys.argv:
        logger.addHandler(logging.StreamHandler())
        sys.exit(main())

    daemon = Daemon("/tmp/dreampi.pid", main)

    if len(sys.argv) == 2:
        if sys.argv[1] == "start":
            daemon.start()
        elif sys.argv[1] == "stop":
            daemon.stop()
        elif sys.argv[1] == "restart":
            daemon.restart()
        else:
            sys.exit(2)
        sys.exit(0)
    else:
        print("Usage: %s start|stop|restart" % sys.argv[0])
        sys.exit(2)
<|endoftext|>