<commit_before># Copyright (C) 2009 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re

import common

class EdifyGenerator(object):
  """Class to generate scripts in the 'edify' recovery script language
  used from donut onwards."""

  def __init__(self, version, info):
    self.script = []
    self.mounts = set()
    self.version = version
    self.info = info

  def MakeTemporary(self):
    """Make a temporary script object whose commands can latter be
    appended to the parent script with AppendScript().  Used when the
    caller wants to generate script commands out-of-order."""
    x = EdifyGenerator(self.version, self.info)
    x.mounts = self.mounts
    return x

  @staticmethod
  def _WordWrap(cmd, linelen=80):
    """'cmd' should be a function call with null characters after each
    parameter (eg, "somefun(foo,\0bar,\0baz)").  This function wraps cmd
    to a given line length, replacing nulls with spaces and/or newlines
    to format it nicely."""
    indent = cmd.index("(")+1
    out = []
    first = True
    x = re.compile("^(.{,%d})\0" % (linelen-indent,))
    while True:
      if not first:
        out.append(" " * indent)
      first = False
      m = x.search(cmd)
      if not m:
        parts = cmd.split("\0", 1)
        out.append(parts[0]+"\n")
        if len(parts) == 1:
          break
        else:
          cmd = parts[1]
          continue
      out.append(m.group(1)+"\n")
      cmd = cmd[m.end():]

    return "".join(out).replace("\0", " ").rstrip("\n")

  def AppendScript(self, other):
    """Append the contents of another script (which should be created
    with temporary=True) to this one."""
    self.script.extend(other.script)

  def AssertSomeFingerprint(self, *fp):
    """Assert that the current system build fingerprint is one of *fp."""
    if not fp:
      raise ValueError("must specify some fingerprints")
    cmd = ('assert(' +
           ' ||\0'.join([('file_getprop("/system/build.prop", '
                         '"ro.build.fingerprint") == "%s"')
                        % i for i in fp]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def AssertOlderBuild(self, timestamp):
    """Assert that the build on the device is older (or the same as)
    the given timestamp."""
    self.script.append(('assert(!less_than_int(%s, '
                        'getprop("ro.build.date.utc")));') % (timestamp,))

  def AssertDevice(self, device):
    """Assert that the device identifier is the given string."""
    cmd = ('assert(getprop("ro.product.device") == "%s" ||\0'
           'getprop("ro.build.product") == "%s");' % (device, device))
    self.script.append(self._WordWrap(cmd))

  def AssertSomeBootloader(self, *bootloaders):
    """Asert that the bootloader version is one of *bootloaders."""
    cmd = ("assert(" +
           " ||\0".join(['getprop("ro.bootloader") == "%s"' % (b,)
                         for b in bootloaders]) +
           ");")
    self.script.append(self._WordWrap(cmd))

  def ShowProgress(self, frac, dur):
    """Update the progress bar, advancing it over 'frac' over the next
    'dur' seconds.  'dur' may be zero to advance it via SetProgress
    commands instead of by time."""
    self.script.append("show_progress(%f, %d);" % (frac, int(dur)))

  def SetProgress(self, frac):
    """Set the position of the progress bar within the chunk defined
    by the most recent ShowProgress call.  'frac' should be in
    [0,1]."""
    self.script.append("set_progress(%f);" % (frac,))

  def PatchCheck(self, filename, *sha1):
    """Check that the given file (or MTD reference) has one of the
    given *sha1 hashes, checking the version saved in cache if the
    file does not match."""
    self.script.append('assert(apply_patch_check("%s"' % (filename,) +
                       "".join([', "%s"' % (i,) for i in sha1]) +
                       '));')

  def FileCheck(self, filename, *sha1):
    """Check that the given file (or MTD reference) has one of the
    given *sha1 hashes."""
    self.script.append('assert(sha1_check(read_file("%s")' % (filename,) +
                       "".join([', "%s"' % (i,) for i in sha1]) +
                       '));')

  def CacheFreeSpaceCheck(self, amount):
    """Check that there's at least 'amount' space that can be made
    available on /cache."""
    self.script.append("assert(apply_patch_space(%d));" % (amount,))

  def Mount(self, mount_point):
    """Mount the partition with the given mount_point."""
    fstab = self.info.get("fstab", None)
    if fstab:
      p = fstab[mount_point]
      self.script.append('mount("%s", "%s", "%s", "%s");' %
                         (p.fs_type, common.PARTITION_TYPES[p.fs_type],
                          p.device, p.mount_point))
      self.mounts.add(p.mount_point)
    else:
      what = mount_point.lstrip("/")
      what = self.info.get("partition_path", "") + what
      self.script.append('mount("%s", "%s", "%s", "%s");' %
                         (self.info["fs_type"], self.info["partition_type"],
                          what, mount_point))
      self.mounts.add(mount_point)

  def UnpackPackageDir(self, src, dst):
    """Unpack a given directory from the OTA package into the given
    destination directory."""
    self.script.append('package_extract_dir("%s", "%s");' % (src, dst))

  def Comment(self, comment):
    """Write a comment into the update script."""
    self.script.append("")
    for i in comment.split("\n"):
      self.script.append("# " + i)
    self.script.append("")

  def Print(self, message):
    """Log a message to the screen (if the logs are visible)."""
    self.script.append('ui_print("%s");' % (message,))

  def FormatPartition(self, partition):
    """Format the given partition, specified by its mount point (eg,
    "/system")."""

    reserve_size = 0
    fstab = self.info.get("fstab", None)
    if fstab:
      p = fstab[partition]
      # Reserve the last 16 Kbytes of an EMMC /data for the crypto footer
      if partition == "/data" and common.PARTITION_TYPES[p.fs_type] == "EMMC":
        reserve_size = -16384
      self.script.append('format("%s", "%s", "%s", "%s");' %
                         (p.fs_type, common.PARTITION_TYPES[p.fs_type], p.device, reserve_size)
    else:
      # older target-files without per-partition types
      partition = self.info.get("partition_path", "") + partition
      self.script.append('format("%s", "%s", "%s", "%s");' %
                         (self.info["fs_type"], self.info["partition_type"],
                          partition, reserve_size))

  def DeleteFiles(self, file_list):
    """Delete all files in file_list."""
    if not file_list: return
    cmd = "delete(" + ",\0".join(['"%s"' % (i,) for i in file_list]) + ");"
    self.script.append(self._WordWrap(cmd))

  def ApplyPatch(self, srcfile, tgtfile, tgtsize, tgtsha1, *patchpairs):
    """Apply binary patches (in *patchpairs) to the given srcfile to
    produce tgtfile (which may be "-" to indicate overwriting the
    source file."""
    if len(patchpairs) % 2 != 0 or len(patchpairs) == 0:
      raise ValueError("bad patches given to ApplyPatch")
    cmd = ['apply_patch("%s",\0"%s",\0%s,\0%d'
           % (srcfile, tgtfile, tgtsha1, tgtsize)]
    for i in range(0, len(patchpairs), 2):
      cmd.append(',\0%s, package_extract_file("%s")' % patchpairs[i:i+2])
    cmd.append(');')
    cmd = "".join(cmd)
    self.script.append(self._WordWrap(cmd))

  def WriteFirmwareImage(self, kind, fn):
    """Arrange to update the given firmware image (kind must be
    "hboot" or "radio") when recovery finishes."""
    if self.version == 1:
      self.script.append(
          ('assert(package_extract_file("%(fn)s", "/tmp/%(kind)s.img"),\n'
           '       write_firmware_image("/tmp/%(kind)s.img", "%(kind)s"));')
          % {'kind': kind, 'fn': fn})
    else:
      self.script.append(
          'write_firmware_image("PACKAGE:%s", "%s");' % (fn, kind))

  def WriteRawImage(self, mount_point, fn):
    """Write the given package file into the partition for the given
    mount point."""

    fstab = self.info["fstab"]
    if fstab:
      p = fstab[mount_point]
      partition_type = common.PARTITION_TYPES[p.fs_type]
      args = {'device': p.device, 'fn': fn}
      if partition_type == "MTD":
        self.script.append(
            ('assert(package_extract_file("%(fn)s", "/tmp/%(device)s.img"),\n'
             '       write_raw_image("/tmp/%(device)s.img", "%(device)s"),\n'
             '       delete("/tmp/%(device)s.img"));') % args)
      elif partition_type == "EMMC":
        self.script.append(
            'package_extract_file("%(fn)s", "%(device)s");' % args)
      else:
        raise ValueError("don't know how to write \"%s\" partitions" % (p.fs_type,))
    else:
      # backward compatibility with older target-files that lack recovery.fstab
      if self.info["partition_type"] == "MTD":
        self.script.append(
            ('assert(package_extract_file("%(fn)s", "/tmp/%(partition)s.img"),\n'
             '       write_raw_image("/tmp/%(partition)s.img", "%(partition)s"),\n'
             '       delete("/tmp/%(partition)s.img"));')
            % {'partition': partition, 'fn': fn})
      elif self.info["partition_type"] == "EMMC":
        self.script.append(
            ('package_extract_file("%(fn)s", "%(dir)s%(partition)s");')
            % {'partition': partition, 'fn': fn,
               'dir': self.info.get("partition_path", ""),
               })
      else:
        raise ValueError("don't know how to write \"%s\" partitions" %
                         (self.info["partition_type"],))

  def SetPermissions(self, fn, uid, gid, mode):
    """Set file ownership and permissions."""
    self.script.append('set_perm(%d, %d, 0%o, "%s");' % (uid, gid, mode, fn))

  def SetPermissionsRecursive(self, fn, uid, gid, dmode, fmode):
    """Recursively set path ownership and permissions."""
    self.script.append('set_perm_recursive(%d, %d, 0%o, 0%o, "%s");'
                       % (uid, gid, dmode, fmode, fn))

  def MakeSymlinks(self, symlink_list):
    """Create symlinks, given a list of (dest, link) pairs."""
    by_dest = {}
    for d, l in symlink_list:
      by_dest.setdefault(d, []).append(l)

    for dest, links in sorted(by_dest.iteritems()):
      cmd = ('symlink("%s", ' % (dest,) +
             ",\0".join(['"' + i + '"' for i in sorted(links)]) + ");")
      self.script.append(self._WordWrap(cmd))

  def RetouchBinaries(self, file_list):
    """Execute the retouch instructions in files listed."""
    cmd = ('retouch_binaries(' +
           ', '.join(['"' + i[0] + '", "' + i[1] + '"' for i in file_list]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def UndoRetouchBinaries(self, file_list):
    """Undo the retouching (retouch to zero offset)."""
    cmd = ('undo_retouch_binaries(' +
           ', '.join(['"' + i[0] + '", "' + i[1] + '"' for i in file_list]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def AppendExtra(self, extra):
    """Append text verbatim to the output script."""
    self.script.append(extra)

  def UnmountAll(self):
    for p in sorted(self.mounts):
      self.script.append('unmount("%s");' % (p,))
    self.mounts = set()

  def AddToZip(self, input_zip, output_zip, input_path=None):
    """Write the accumulated script to the output_zip file.  input_zip
    is used as the source for the 'updater' binary needed to run
    script.  If input_path is not None, it will be used as a local
    path for the binary instead of input_zip."""

    self.UnmountAll()

    common.ZipWriteStr(output_zip, "META-INF/com/google/android/updater-script",
                       "\n".join(self.script) + "\n")

    if input_path is None:
      data = input_zip.read("OTA/bin/updater")
    else:
      data = open(os.path.join(input_path, "updater")).read()
    common.ZipWriteStr(output_zip, "META-INF/com/google/android/update-binary",
                       data, perms=0755)
<commit_msg>am 87762c7e: am b43ac8fc: Fix build break<commit_after># Copyright (C) 2009 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re

import common

class EdifyGenerator(object):
  """Class to generate scripts in the 'edify' recovery script language
  used from donut onwards."""

  def __init__(self, version, info):
    self.script = []
    self.mounts = set()
    self.version = version
    self.info = info

  def MakeTemporary(self):
    """Make a temporary script object whose commands can latter be
    appended to the parent script with AppendScript().  Used when the
    caller wants to generate script commands out-of-order."""
    x = EdifyGenerator(self.version, self.info)
    x.mounts = self.mounts
    return x

  @staticmethod
  def _WordWrap(cmd, linelen=80):
    """'cmd' should be a function call with null characters after each
    parameter (eg, "somefun(foo,\0bar,\0baz)").  This function wraps cmd
    to a given line length, replacing nulls with spaces and/or newlines
    to format it nicely."""
    indent = cmd.index("(")+1
    out = []
    first = True
    x = re.compile("^(.{,%d})\0" % (linelen-indent,))
    while True:
      if not first:
        out.append(" " * indent)
      first = False
      m = x.search(cmd)
      if not m:
        parts = cmd.split("\0", 1)
        out.append(parts[0]+"\n")
        if len(parts) == 1:
          break
        else:
          cmd = parts[1]
          continue
      out.append(m.group(1)+"\n")
      cmd = cmd[m.end():]

    return "".join(out).replace("\0", " ").rstrip("\n")

  def AppendScript(self, other):
    """Append the contents of another script (which should be created
    with temporary=True) to this one."""
    self.script.extend(other.script)

  def AssertSomeFingerprint(self, *fp):
    """Assert that the current system build fingerprint is one of *fp."""
    if not fp:
      raise ValueError("must specify some fingerprints")
    cmd = ('assert(' +
           ' ||\0'.join([('file_getprop("/system/build.prop", '
                         '"ro.build.fingerprint") == "%s"')
                        % i for i in fp]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def AssertOlderBuild(self, timestamp):
    """Assert that the build on the device is older (or the same as)
    the given timestamp."""
    self.script.append(('assert(!less_than_int(%s, '
                        'getprop("ro.build.date.utc")));') % (timestamp,))

  def AssertDevice(self, device):
    """Assert that the device identifier is the given string."""
    cmd = ('assert(getprop("ro.product.device") == "%s" ||\0'
           'getprop("ro.build.product") == "%s");' % (device, device))
    self.script.append(self._WordWrap(cmd))

  def AssertSomeBootloader(self, *bootloaders):
    """Asert that the bootloader version is one of *bootloaders."""
    cmd = ("assert(" +
           " ||\0".join(['getprop("ro.bootloader") == "%s"' % (b,)
                         for b in bootloaders]) +
           ");")
    self.script.append(self._WordWrap(cmd))

  def ShowProgress(self, frac, dur):
    """Update the progress bar, advancing it over 'frac' over the next
    'dur' seconds.  'dur' may be zero to advance it via SetProgress
    commands instead of by time."""
    self.script.append("show_progress(%f, %d);" % (frac, int(dur)))

  def SetProgress(self, frac):
    """Set the position of the progress bar within the chunk defined
    by the most recent ShowProgress call.  'frac' should be in
    [0,1]."""
    self.script.append("set_progress(%f);" % (frac,))

  def PatchCheck(self, filename, *sha1):
    """Check that the given file (or MTD reference) has one of the
    given *sha1 hashes, checking the version saved in cache if the
    file does not match."""
    self.script.append('assert(apply_patch_check("%s"' % (filename,) +
                       "".join([', "%s"' % (i,) for i in sha1]) +
                       '));')

  def FileCheck(self, filename, *sha1):
    """Check that the given file (or MTD reference) has one of the
    given *sha1 hashes."""
    self.script.append('assert(sha1_check(read_file("%s")' % (filename,) +
                       "".join([', "%s"' % (i,) for i in sha1]) +
                       '));')

  def CacheFreeSpaceCheck(self, amount):
    """Check that there's at least 'amount' space that can be made
    available on /cache."""
    self.script.append("assert(apply_patch_space(%d));" % (amount,))

  def Mount(self, mount_point):
    """Mount the partition with the given mount_point."""
    fstab = self.info.get("fstab", None)
    if fstab:
      p = fstab[mount_point]
      self.script.append('mount("%s", "%s", "%s", "%s");' %
                         (p.fs_type, common.PARTITION_TYPES[p.fs_type],
                          p.device, p.mount_point))
      self.mounts.add(p.mount_point)
    else:
      what = mount_point.lstrip("/")
      what = self.info.get("partition_path", "") + what
      self.script.append('mount("%s", "%s", "%s", "%s");' %
                         (self.info["fs_type"], self.info["partition_type"],
                          what, mount_point))
      self.mounts.add(mount_point)

  def UnpackPackageDir(self, src, dst):
    """Unpack a given directory from the OTA package into the given
    destination directory."""
    self.script.append('package_extract_dir("%s", "%s");' % (src, dst))

  def Comment(self, comment):
    """Write a comment into the update script."""
    self.script.append("")
    for i in comment.split("\n"):
      self.script.append("# " + i)
    self.script.append("")

  def Print(self, message):
    """Log a message to the screen (if the logs are visible)."""
    self.script.append('ui_print("%s");' % (message,))

  def FormatPartition(self, partition):
    """Format the given partition, specified by its mount point (eg,
    "/system")."""

    reserve_size = 0
    fstab = self.info.get("fstab", None)
    if fstab:
      p = fstab[partition]
      # Reserve the last 16 Kbytes of an EMMC /data for the crypto footer
      if partition == "/data" and common.PARTITION_TYPES[p.fs_type] == "EMMC":
        reserve_size = -16384
      self.script.append('format("%s", "%s", "%s", "%s");' %
                         (p.fs_type, common.PARTITION_TYPES[p.fs_type], p.device, reserve_size))
    else:
      # older target-files without per-partition types
      partition = self.info.get("partition_path", "") + partition
      self.script.append('format("%s", "%s", "%s", "%s");' %
                         (self.info["fs_type"], self.info["partition_type"],
                          partition, reserve_size))

  def DeleteFiles(self, file_list):
    """Delete all files in file_list."""
    if not file_list: return
    cmd = "delete(" + ",\0".join(['"%s"' % (i,) for i in file_list]) + ");"
    self.script.append(self._WordWrap(cmd))

  def ApplyPatch(self, srcfile, tgtfile, tgtsize, tgtsha1, *patchpairs):
    """Apply binary patches (in *patchpairs) to the given srcfile to
    produce tgtfile (which may be "-" to indicate overwriting the
    source file."""
    if len(patchpairs) % 2 != 0 or len(patchpairs) == 0:
      raise ValueError("bad patches given to ApplyPatch")
    cmd = ['apply_patch("%s",\0"%s",\0%s,\0%d'
           % (srcfile, tgtfile, tgtsha1, tgtsize)]
    for i in range(0, len(patchpairs), 2):
      cmd.append(',\0%s, package_extract_file("%s")' % patchpairs[i:i+2])
    cmd.append(');')
    cmd = "".join(cmd)
    self.script.append(self._WordWrap(cmd))

  def WriteFirmwareImage(self, kind, fn):
    """Arrange to update the given firmware image (kind must be
    "hboot" or "radio") when recovery finishes."""
    if self.version == 1:
      self.script.append(
          ('assert(package_extract_file("%(fn)s", "/tmp/%(kind)s.img"),\n'
           '       write_firmware_image("/tmp/%(kind)s.img", "%(kind)s"));')
          % {'kind': kind, 'fn': fn})
    else:
      self.script.append(
          'write_firmware_image("PACKAGE:%s", "%s");' % (fn, kind))

  def WriteRawImage(self, mount_point, fn):
    """Write the given package file into the partition for the given
    mount point."""

    fstab = self.info["fstab"]
    if fstab:
      p = fstab[mount_point]
      partition_type = common.PARTITION_TYPES[p.fs_type]
      args = {'device': p.device, 'fn': fn}
      if partition_type == "MTD":
        self.script.append(
            ('assert(package_extract_file("%(fn)s", "/tmp/%(device)s.img"),\n'
             '       write_raw_image("/tmp/%(device)s.img", "%(device)s"),\n'
             '       delete("/tmp/%(device)s.img"));') % args)
      elif partition_type == "EMMC":
        self.script.append(
            'package_extract_file("%(fn)s", "%(device)s");' % args)
      else:
        raise ValueError("don't know how to write \"%s\" partitions" % (p.fs_type,))
    else:
      # backward compatibility with older target-files that lack recovery.fstab
      if self.info["partition_type"] == "MTD":
        self.script.append(
            ('assert(package_extract_file("%(fn)s", "/tmp/%(partition)s.img"),\n'
             '       write_raw_image("/tmp/%(partition)s.img", "%(partition)s"),\n'
             '       delete("/tmp/%(partition)s.img"));')
            % {'partition': partition, 'fn': fn})
      elif self.info["partition_type"] == "EMMC":
        self.script.append(
            ('package_extract_file("%(fn)s", "%(dir)s%(partition)s");')
            % {'partition': partition, 'fn': fn,
               'dir': self.info.get("partition_path", ""),
               })
      else:
        raise ValueError("don't know how to write \"%s\" partitions" %
                         (self.info["partition_type"],))

  def SetPermissions(self, fn, uid, gid, mode):
    """Set file ownership and permissions."""
    self.script.append('set_perm(%d, %d, 0%o, "%s");' % (uid, gid, mode, fn))

  def SetPermissionsRecursive(self, fn, uid, gid, dmode, fmode):
    """Recursively set path ownership and permissions."""
    self.script.append('set_perm_recursive(%d, %d, 0%o, 0%o, "%s");'
                       % (uid, gid, dmode, fmode, fn))

  def MakeSymlinks(self, symlink_list):
    """Create symlinks, given a list of (dest, link) pairs."""
    by_dest = {}
    for d, l in symlink_list:
      by_dest.setdefault(d, []).append(l)

    for dest, links in sorted(by_dest.iteritems()):
      cmd = ('symlink("%s", ' % (dest,) +
             ",\0".join(['"' + i + '"' for i in sorted(links)]) + ");")
      self.script.append(self._WordWrap(cmd))

  def RetouchBinaries(self, file_list):
    """Execute the retouch instructions in files listed."""
    cmd = ('retouch_binaries(' +
           ', '.join(['"' + i[0] + '", "' + i[1] + '"' for i in file_list]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def UndoRetouchBinaries(self, file_list):
    """Undo the retouching (retouch to zero offset)."""
    cmd = ('undo_retouch_binaries(' +
           ', '.join(['"' + i[0] + '", "' + i[1] + '"' for i in file_list]) +
           ');')
    self.script.append(self._WordWrap(cmd))

  def AppendExtra(self, extra):
    """Append text verbatim to the output script."""
    self.script.append(extra)

  def UnmountAll(self):
    for p in sorted(self.mounts):
      self.script.append('unmount("%s");' % (p,))
    self.mounts = set()

  def AddToZip(self, input_zip, output_zip, input_path=None):
    """Write the accumulated script to the output_zip file.  input_zip
    is used as the source for the 'updater' binary needed to run
    script.  If input_path is not None, it will be used as a local
    path for the binary instead of input_zip."""

    self.UnmountAll()

    common.ZipWriteStr(output_zip, "META-INF/com/google/android/updater-script",
                       "\n".join(self.script) + "\n")

    if input_path is None:
      data = input_zip.read("OTA/bin/updater")
    else:
      data = open(os.path.join(input_path, "updater")).read()
    common.ZipWriteStr(output_zip, "META-INF/com/google/android/update-binary",
                       data, perms=0755)
<|endoftext|>


<commit_before><commit_msg>Create NetMRI - BackUp Configuration of Unsupported Device.py<commit_after><|endoftext|>


<commit_before>"""For internal use only. It provides a slice-like file reader."""

import os

try:
    # pylint: disable=no-name-in-module
    from multiprocessing import Lock
except ImportError:
    from threading import Lock

class FileBuffer(object):
    """A slice-able file reader"""
    def __init__(self, database):
        self._handle = open(database, 'rb')
        self._size = os.fstat(self._handle.fileno()).st_size
        if not hasattr(os, 'pread'):
            self._lock = Lock()

    def __getitem__(self, key):
        if isinstance(key, slice):
            return self._read(key.stop - key.start, key.start)
        if isinstance(key, int):
            return self._read(1, key)
        raise TypeError("Invalid argument type.")

    def rfind(self, needle, start):
        """Reverse find needle from start"""
        pos = self._read(self._size - start - 1, start).rfind(needle)
        if pos == -1:
            return pos
        return start + pos

    def size(self):
        """Size of file"""
        return self._size

    def close(self):
        """Close file"""
        self._handle.close()

    if hasattr(os, 'pread'):

        def _read(self, buffersize, offset):
            """read that uses pread"""
            # pylint: disable=no-member
            return os.pread(self._handle.fileno(), buffersize, offset)

    else:

        def _read(self, buffersize, offset):
            """read with a lock

            This lock is necessary as after a fork, the different processes
            will share the same file table entry, even if we dup the fd, and
            as such the same offsets. There does not appear to be a way to
            duplicate the file table entry and we cannot re-open based on the
            original path as that file may have replaced with another or
            unlinked.
            """
            with self._lock:
                self._handle.seek(offset)
                return self._handle.read(buffersize)
<commit_msg>Make FileBuffer behave more like other buffers<commit_after>"""For internal use only. It provides a slice-like file reader."""

import os

try:
    # pylint: disable=no-name-in-module
    from multiprocessing import Lock
except ImportError:
    from threading import Lock

class FileBuffer(object):
    """A slice-able file reader"""
    def __init__(self, database):
        self._handle = open(database, 'rb')
        self._size = os.fstat(self._handle.fileno()).st_size
        if not hasattr(os, 'pread'):
            self._lock = Lock()

    def __getitem__(self, key):
        if isinstance(key, slice):
            return self._read(key.stop - key.start, key.start)
        if isinstance(key, int):
            return self._read(1, key)[0]
        raise TypeError("Invalid argument type.")

    def rfind(self, needle, start):
        """Reverse find needle from start"""
        pos = self._read(self._size - start - 1, start).rfind(needle)
        if pos == -1:
            return pos
        return start + pos

    def size(self):
        """Size of file"""
        return self._size

    def close(self):
        """Close file"""
        self._handle.close()

    if hasattr(os, 'pread'):

        def _read(self, buffersize, offset):
            """read that uses pread"""
            # pylint: disable=no-member
            return os.pread(self._handle.fileno(), buffersize, offset)

    else:

        def _read(self, buffersize, offset):
            """read with a lock

            This lock is necessary as after a fork, the different processes
            will share the same file table entry, even if we dup the fd, and
            as such the same offsets. There does not appear to be a way to
            duplicate the file table entry and we cannot re-open based on the
            original path as that file may have replaced with another or
            unlinked.
            """
            with self._lock:
                self._handle.seek(offset)
                return self._handle.read(buffersize)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# Copyright 2017, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import multiprocessing
import os
import subprocess
import sys
import argparse

import python_utils.jobset as jobset
import python_utils.start_port_server as start_port_server

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
if not os.path.exists('reports'):
  os.makedirs('reports')

port_server_port = 32766
start_port_server.start_port_server(port_server_port)

def fnize(s):
  out = ''
  for c in s:
    if c in '<>, /':
      if len(out) and out[-1] == '_': continue
      out += '_'
    else:
      out += c
  return out

# index html
index_html = """

<title>Microbenchmark Results</title>

"""

def heading(name):
  global index_html
  index_html += "
%s
\n" % name

def link(txt, tgt):
  global index_html
  index_html += "
%s
\n" % (tgt, txt)

def text(txt):
  global index_html
  index_html += "

%s

" % txt

def collect_latency(bm_name, args):
  """generate latency profiles"""
  benchmarks = []
  profile_analysis = []
  cleanup = []

  heading('Latency Profiles: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
  for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
                                       '--benchmark_list_tests']).splitlines():
    link(line, '%s.txt' % fnize(line))
    benchmarks.append(
        jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line],
                       environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))
    profile_analysis.append(
        jobset.JobSpec([sys.executable,
                        'tools/profiling/latency_profile/profile_analyzer.py',
                        '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
                        '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))
    cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
    # periodically flush out the list of jobs: profile_analysis jobs at least
    # consume upwards of five gigabytes of ram in some cases, and so analysing
    # hundreds of them at once is impractical -- but we want at least some
    # concurrency or the work takes too long
    if len(benchmarks) >= min(4, multiprocessing.cpu_count()):
      # run up to half the cpu count: each benchmark can use up to two cores
      # (one for the microbenchmark, one for the data flush)
      jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
                 add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
      jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
      jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
      benchmarks = []
      profile_analysis = []
      cleanup = []
  # run the remaining benchmarks that weren't flushed
  if len(benchmarks):
    jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
               add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
    jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
    jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

def collect_perf(bm_name, args):
  """generate flamegraphs"""
  heading('Flamegraphs: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
  for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
                                       '--benchmark_list_tests']).splitlines():
    subprocess.check_call(['sudo', 'perf', 'record', '-g', '-c', '1000',
                           'bins/mutrace/%s' % bm_name,
                           '--benchmark_filter=^%s$' % line,
                           '--benchmark_min_time=20'])
    with open('/tmp/bm.perf', 'w') as f:
      f.write(subprocess.check_output(['sudo', 'perf', 'script']))
    with open('/tmp/bm.folded', 'w') as f:
      f.write(subprocess.check_output([
          '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf']))
    link(line, '%s.svg' % fnize(line))
    with open('reports/%s.svg' % fnize(line), 'w') as f:
      f.write(subprocess.check_output([
          '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded']))

def collect_summary(bm_name, args):
  heading('Summary: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=counters', '-j', '%d' % multiprocessing.cpu_count()])
  text(subprocess.check_output(['bins/counters/%s' % bm_name,
                                '--benchmark_out=out.json',
                                '--benchmark_out_format=json']))
  if args.bigquery_upload:
    with open('/tmp/out.csv', 'w') as f:
      f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json']))
    subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv'])

collectors = {
  'latency': collect_latency,
  'perf': collect_perf,
  'summary': collect_summary,
}

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
argp.add_argument('-c', '--collect',
                  choices=sorted(collectors.keys()),
                  nargs='+',
                  default=sorted(collectors.keys()),
                  help='Which collectors should be run against each benchmark')
argp.add_argument('-b', '--benchmarks',
                  default=['bm_fullstack'],
                  nargs='+',
                  type=str,
                  help='Which microbenchmarks should be run')
argp.add_argument('--bigquery_upload',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Upload results from summary collection to bigquery')
args = argp.parse_args()

for bm_name in args.benchmarks:
  for collect in args.collect:
    collectors[collect](bm_name, args)

index_html += "\n\n"
with open('reports/index.html', 'w') as f:
  f.write(index_html)
<commit_msg>Add \n<commit_after>#!/usr/bin/env python2.7
# Copyright 2017, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import multiprocessing
import os
import subprocess
import sys
import argparse

import python_utils.jobset as jobset
import python_utils.start_port_server as start_port_server

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
if not os.path.exists('reports'):
  os.makedirs('reports')

port_server_port = 32766
start_port_server.start_port_server(port_server_port)

def fnize(s):
  out = ''
  for c in s:
    if c in '<>, /':
      if len(out) and out[-1] == '_': continue
      out += '_'
    else:
      out += c
  return out

# index html
index_html = """

<title>Microbenchmark Results</title>

"""

def heading(name):
  global index_html
  index_html += "
%s
\n" % name

def link(txt, tgt):
  global index_html
  index_html += "
%s
\n" % (tgt, txt)

def text(txt):
  global index_html
  index_html += "

%s

\n" % txt

def collect_latency(bm_name, args):
  """generate latency profiles"""
  benchmarks = []
  profile_analysis = []
  cleanup = []

  heading('Latency Profiles: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
  for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
                                       '--benchmark_list_tests']).splitlines():
    link(line, '%s.txt' % fnize(line))
    benchmarks.append(
        jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line],
                       environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))
    profile_analysis.append(
        jobset.JobSpec([sys.executable,
                        'tools/profiling/latency_profile/profile_analyzer.py',
                        '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
                        '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))
    cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
    # periodically flush out the list of jobs: profile_analysis jobs at least
    # consume upwards of five gigabytes of ram in some cases, and so analysing
    # hundreds of them at once is impractical -- but we want at least some
    # concurrency or the work takes too long
    if len(benchmarks) >= min(4, multiprocessing.cpu_count()):
      # run up to half the cpu count: each benchmark can use up to two cores
      # (one for the microbenchmark, one for the data flush)
      jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
                 add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
      jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
      jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
      benchmarks = []
      profile_analysis = []
      cleanup = []
  # run the remaining benchmarks that weren't flushed
  if len(benchmarks):
    jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
               add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
    jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
    jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

def collect_perf(bm_name, args):
  """generate flamegraphs"""
  heading('Flamegraphs: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
  for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
                                       '--benchmark_list_tests']).splitlines():
    subprocess.check_call(['sudo', 'perf', 'record', '-g', '-c', '1000',
                           'bins/mutrace/%s' % bm_name,
                           '--benchmark_filter=^%s$' % line,
                           '--benchmark_min_time=20'])
    with open('/tmp/bm.perf', 'w') as f:
      f.write(subprocess.check_output(['sudo', 'perf', 'script']))
    with open('/tmp/bm.folded', 'w') as f:
      f.write(subprocess.check_output([
          '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf']))
    link(line, '%s.svg' % fnize(line))
    with open('reports/%s.svg' % fnize(line), 'w') as f:
      f.write(subprocess.check_output([
          '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded']))

def collect_summary(bm_name, args):
  heading('Summary: %s' % bm_name)
  subprocess.check_call(
      ['make', bm_name,
       'CONFIG=counters', '-j', '%d' % multiprocessing.cpu_count()])
  text(subprocess.check_output(['bins/counters/%s' % bm_name,
                                '--benchmark_out=out.json',
                                '--benchmark_out_format=json']))
  if args.bigquery_upload:
    with open('/tmp/out.csv', 'w') as f:
      f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json']))
    subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv'])

collectors = {
  'latency': collect_latency,
  'perf': collect_perf,
  'summary': collect_summary,
}

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
argp.add_argument('-c', '--collect',
                  choices=sorted(collectors.keys()),
                  nargs='+',
                  default=sorted(collectors.keys()),
                  help='Which collectors should be run against each benchmark')
argp.add_argument('-b', '--benchmarks',
                  default=['bm_fullstack'],
                  nargs='+',
                  type=str,
                  help='Which microbenchmarks should be run')
argp.add_argument('--bigquery_upload',
                  default=False,
                  action='store_const',
                  const=True,
                  help='Upload results from summary collection to bigquery')
args = argp.parse_args()

for bm_name in args.benchmarks:
  for collect in args.collect:
    collectors[collect](bm_name, args)

index_html += "\n\n"
with open('reports/index.html', 'w') as f:
  f.write(index_html)
<|endoftext|>


<commit_before>#! python
import urllib.request
import urllib.parse
import codecs
import json
import random
import datetime
import time
import logging
import requests
import shutil
import os.path
from fake_useragent import UserAgent
import re

try:
    from config import token
except:
    print("You must create a config.py file with your bot token in it!")

logging.basicConfig(filename='ohaibot.log',
                    format='%(asctime)s %(levelname)s %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO)

bot_url = 'https://api.telegram.org/bot' + token + '/'
offset_file = 'offset.txt'
update_id = 0

# Map of keywords to URLs, used for static content (non-searched)
keyword_map = { 'cantbrain': 'https://dl.dropboxusercontent.com/u/11466/meme/cantbrain.jpg',
                'fixit': 'https://dl.dropboxusercontent.com/u/11466/gifs/fixit.gif',
                'trololo': 'https://dl.dropboxusercontent.com/u/11466/meme/trolololo.jpg',
                'noideadog': 'https://dl.dropboxusercontent.com/u/11466/meme/ihavenoidea.jpg',
                'afraidtoask': 'https://dl.dropboxusercontent.com/u/11466/meme/noideaafraidtoask.jpg',
                'climbingnope': 'https://dl.dropboxusercontent.com/u/11466/meme/towerclimbing.gif',
                'nopenopenope': 'https://dl.dropboxusercontent.com/u/11466/gifs/nopenopenope.gif',
                'ketchup': 'https://dl.dropboxusercontent.com/u/11466/gifs/ketchup.gif',
                'lastf': 'https://dl.dropboxusercontent.com/u/11466/gifs/last_f.gif',
                'deadhorse': 'https://dl.dropboxusercontent.com/u/11466/gifs/beatADeadHorse.gif',
                'billandsteve': 'https://dl.dropboxusercontent.com/u/11466/gifs/billandsteve.gif',
                'duckno': 'https://dl.dropboxusercontent.com/u/11466/gifs/duckno.gif',
                'frakthis': 'https://dl.dropboxusercontent.com/u/11466/gifs/frakthisfrakthat.gif',
                'why': 'https://dl.dropboxusercontent.com/u/11466/gifs/WHY.gif',
                'sillywalknope': 'https://dl.dropboxusercontent.com/u/11466/gifs/SillyWalkNope.gif',
                'sure': 'https://dl.dropboxusercontent.com/u/11466/gifs/sure_i_tested_it.gif',
                'maxpower': 'https://dl.dropboxusercontent.com/u/11466/meme/maxpower.jpg',
                'rustled': 'https://dl.dropboxusercontent.com/u/11466/meme/jimmiesrustled.png',
                'overrustled': 'https://dl.dropboxusercontent.com/u/11466/meme/jimmiesrustledmaximum.jpg',
                'random': 'http://www.reddit.com/r/random',
                }

'''Send a text only message, useful for help, feedback and errors'''
def send_simple_message(chat_id, text):
    try:
        data = urllib.parse.urlencode({'chat_id': chat_id,
                                       'text': text})
        urllib.request.urlopen(bot_url + 'sendMessage', data.encode('utf-8'))
    except Exception as e:
        logging.critical('Something went wrong when sending simple message: ' + str(e))

''' Required for gifs, otherwise they will not be animated'''
def send_document(chat_id, file_name):
    files = {'document': open(file_name, 'rb')}
    data = {'chat_id': str(chat_id)}
    logging.info(chat_id)
    try:
        r = requests.post(bot_url + 'sendDocument', data=data, files=files)
    except:
        logging.critical('failed to upload file!')
        logging.critical(r.text)

''' For sending photos only. If gif, use send_document'''
def send_photo(chat_id, file_name):
    files = {'photo': open(file_name, 'rb')}
    data = {'chat_id': str(chat_id)}
    logging.info(chat_id)
    try:
        r = requests.post(bot_url + 'sendPhoto', data=data, files=files)
    except:
        logging.critical('failed to upload file!')
        logging.critical(r.text)

'''Images on google like to use redirects, so get the destination url to use'''
def get_redirect_url(url):
    try:
        response = requests.get(url)
        actual_url = response.url
    except:
        logging.info("Couldn't get URL")
        actual_url = url

    # Lets try to clean the URL up as well by removing all but netloc and path
    url_object = urllib.parse.urlparse(actual_url)
    url = url_object.scheme + "://" + url_object.netloc + url_object.path

    return url

'''Search for an image on google. Grabs the first one, if it ends in html or php
 grab the next one in the array'''
def image_search(search_term):
    search_term = re.sub(r'\W+', '', search_term)
    url = "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + search_term + "&start=0&safe=active"
    response = None
    try:
        response = requests.get(url).json()
    except:
        response = None

    if response and len(response['responseData']['results']) > 0:
        logging.info("We have %s results, yay!" % len(response['responseData']['results']) )
        for result in response['responseData']['results']:
            image_url = result['unescapedUrl']
            real_url = get_redirect_url(image_url)
            if real_url.endswith('html') or real_url.endswith('php') or real_url.endswith('htm'):
                logging.info("Could not get image url for %s" % real_url)
                continue
            else:
                return_value = real_url
                break
    else:
        logging.info("Couldn't find any results! *shrugs*")
        return_value = None

    logging.debug("return_value: %s" % return_value)
    return return_value

''' Downloads the given file url to the cache folder'''
def download_file(url):
    file_name = url.split('/')[-1]
    ua = UserAgent()
    # todo create cache folder if it does not already exist.
    cache_folder = 'cache'
    if os.path.isfile(os.path.join(cache_folder, file_name)):
        logging.info('File already exists, skipping download')
        return os.path.join(cache_folder, file_name)
    else:
        logging.info("Downloading %s" % url)
        response = None
        try:
            response = requests.get(url, stream=True, headers = {'User-Agent': ua.chrome } )
        except:
            resposne = None

        fail = False
        try:
            if response.status_code == 200:
                with open(os.path.join('cache', file_name), 'wb') as out_file:
                    for chunk in response.iter_content(1024):
                        out_file.write(chunk)
                    else:
                        logging.critical("Couldn't download %s" % file_name)
        except:
            fail = True
        del response
        if not fail:
            file_path = os.path.join(cache_folder, file_name)
            logging.info("Downloaded %s" % file_path)
            return file_path
        else:
            return None

def get_help(chat_id):
    return_message = 'Query Options:\n\n'
    return_message = return_message + '/get search for this' + '\n\n'

    return_message = return_message + 'Command Options:\n\n'
    for item in sorted(keyword_map):
        return_message = return_message + '/' + item + '\n'

    send_simple_message(chat_id, return_message)

def get_image(chat_id, messagetext):
    message_parts = messagetext.split(' ')
    message_parts.pop(0)
    search_terms = ' '.join(message_parts)
    image_url = image_search(search_terms)
    if image_url:
        logging.debug("Supposedly URL: %s" % image_url)
        file_name = download_file(image_url)
        if not file_name:
            logging.critical("%s does not exist" % file_name)

        logging.debug("file: %s" % file_name)
        # Try to be smart about the content type
        try:
            if file_name.endswith('gif'):
                send_document(chat_id, file_name)
            elif file_name.endswith('png') or file_name.endswith('jpg') or file_name.endswith('jpeg'):
                send_photo(chat_id, file_name)
                return True
            else:
                send_simple_message(chat_id, "I have failed to find a picture for %s." % search_terms)
            return None
        except:
            send_simple_message(chat_id, "I have failed to find a picture for %s." % search_terms)
            return None
    else:
        send_simple_message(chat_id, "I have failed to find a picture for %s." % search_terms)
        return None

def get_static(chat_id, messagetext):
    command = messagetext.strip('/')
    if command in keyword_map:
        file_ext = keyword_map[command].split('.')[-1]
        if file_ext in ['jpg', 'jpeg', 'png']:
            file_name = download_file(keyword_map[command])
            logging.info("Sending photo %s" % file_name)
            send_photo(chat_id, file_name)
            return True
        elif file_ext in ['gif']:
            file_name = download_file(keyword_map[command])
            logging.info("Sending gif %s" % file_name)
            send_document(chat_id, file_name)
            return True
        else:
            return_message = get_redirect_url(keyword_map[command])
            send_simple_message(chat_id, return_message)
            return True
    else:
        return None

'''All bot logic happens here and calls out to functions'''
def do_bot_stuff(update_id):
    try:
        data = urllib.parse.urlencode({'offset': format(update_id),
                                       'limit': '100', 'timeout': '60'})

        response = urllib.request.urlopen(bot_url + 'getUpdates',
                                          data.encode('utf-8'))

        reader = codecs.getreader("utf-8")
        data = json.load(reader(response))
    except Exception as e:
        logging.warning('Something went wrong when fetching update: ' + str(e))
        return update_id

    if data['ok'] == True:
        for update in data['result']:

            # take new update id
            update_id = update['update_id'] + 1
            message = update['message']

            # respond if this is a message containing text
            if 'text' in message:
                # Get text of the message
                messagetext = str(message['text'])

                # Skip anything that isn't a slash command
                if not messagetext.startswith('/'):
                    continue

                # Get chat id, which is used to send the return message
                chat_id = message['chat']['id']

                # Show help text
                if messagetext == '/help':
                    get_help(chat_id)
                # get a image via google image search API
                elif messagetext.startswith('/get'):
                    result = get_image(chat_id, messagetext)
                    logging.info("Got Image: %s" % result)
                # Is it a statically set item?
                else:
                    result =  get_static(chat_id, messagetext)
                    logging.info("Got Image: %s" % result)
                    continue

    return update_id

'''Main program loop, handles timing and long polling'''
def main():
    # make sure the offset file exists and contains an integer
    try:
        file = open(offset_file, 'rt')
        update_id = int(file.read())
        file.close()
    except Exception as e:
        logging.warning(offset_file + 'does not exist. Creating it now' + str(e))
        with open(offset_file, 'w') as file:
            file.write('0')
            update_id = 0
    # main program loop
    while True:
    # process updates
        newupdate_id = do_bot_stuff(update_id)

        # Write the update ID to a file and sleep 3 seconds if we processed updates
        if (newupdate_id != update_id):
            file = open(offset_file, 'wt')
            file.write(str(newupdate_id))
            file.close()
            time.sleep(3)
        else:
            # Otherwise we can wait some more during long polling if we have to.
            time.sleep(1)

        # use new update ID
        update_id = newupdate_id

if __name__ == '__main__':
    main()
<commit_msg>Added ability to try next result if the first one fails for some reason<commit_after>#! python
import urllib.request
import urllib.parse
import codecs
import json
import random
import datetime
import time
import logging
import requests
import shutil
import os.path
from fake_useragent import UserAgent
import re

try:
    from config import token
except:
    print("You must create a config.py file with your bot token in it!")

logging.basicConfig(filename='ohaibot.log',
                    format='%(asctime)s %(levelname)s %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO)

bot_url = 'https://api.telegram.org/bot' + token + '/'
offset_file = 'offset.txt'
update_id = 0

# Map of keywords to URLs, used for static content (non-searched)
keyword_map = { 'cantbrain': 'https://dl.dropboxusercontent.com/u/11466/meme/cantbrain.jpg',
                'fixit': 'https://dl.dropboxusercontent.com/u/11466/gifs/fixit.gif',
                'trololo': 'https://dl.dropboxusercontent.com/u/11466/meme/trolololo.jpg',
                'noideadog': 'https://dl.dropboxusercontent.com/u/11466/meme/ihavenoidea.jpg',
                'afraidtoask': 'https://dl.dropboxusercontent.com/u/11466/meme/noideaafraidtoask.jpg',
                'climbingnope': 'https://dl.dropboxusercontent.com/u/11466/meme/towerclimbing.gif',
                'nopenopenope': 'https://dl.dropboxusercontent.com/u/11466/gifs/nopenopenope.gif',
                'ketchup': 'https://dl.dropboxusercontent.com/u/11466/gifs/ketchup.gif',
                'lastf': 'https://dl.dropboxusercontent.com/u/11466/gifs/last_f.gif',
                'deadhorse': 'https://dl.dropboxusercontent.com/u/11466/gifs/beatADeadHorse.gif',
                'billandsteve': 'https://dl.dropboxusercontent.com/u/11466/gifs/billandsteve.gif',
                'duckno': 'https://dl.dropboxusercontent.com/u/11466/gifs/duckno.gif',
                'frakthis': 'https://dl.dropboxusercontent.com/u/11466/gifs/frakthisfrakthat.gif',
                'why': 'https://dl.dropboxusercontent.com/u/11466/gifs/WHY.gif',
                'sillywalknope': 'https://dl.dropboxusercontent.com/u/11466/gifs/SillyWalkNope.gif',
                'sure': 'https://dl.dropboxusercontent.com/u/11466/gifs/sure_i_tested_it.gif',
                'maxpower': 'https://dl.dropboxusercontent.com/u/11466/meme/maxpower.jpg',
                'rustled': 'https://dl.dropboxusercontent.com/u/11466/meme/jimmiesrustled.png',
                'overrustled': 'https://dl.dropboxusercontent.com/u/11466/meme/jimmiesrustledmaximum.jpg',
                'random': 'http://www.reddit.com/r/random',
                }

'''Send a text only message, useful for help, feedback and errors'''
def send_simple_message(chat_id, text):
    try:
        data = urllib.parse.urlencode({'chat_id': chat_id,
                                       'text': text})
        urllib.request.urlopen(bot_url + 'sendMessage', data.encode('utf-8'))
    except Exception as e:
        logging.critical('Something went wrong when sending simple message: ' + str(e))

''' Required for gifs, otherwise they will not be animated'''
def send_document(chat_id, file_name):
    files = {'document': open(file_name, 'rb')}
    data = {'chat_id': str(chat_id)}
    logging.info(chat_id)
    try:
        r = requests.post(bot_url + 'sendDocument', data=data, files=files)
    except:
        logging.critical('failed to upload file!')
        logging.critical(r.text)

''' For sending photos only. If gif, use send_document'''
def send_photo(chat_id, file_name):
    files = {'photo': open(file_name, 'rb')}
    data = {'chat_id': str(chat_id)}
    logging.info(chat_id)
    try:
        r = requests.post(bot_url + 'sendPhoto', data=data, files=files)
    except:
        logging.critical('failed to upload file!')
        logging.critical(r.text)

'''Images on google like to use redirects, so get the destination url to use'''
def get_redirect_url(url):
    try:
        response = requests.get(url)
        actual_url = response.url
    except:
        logging.info("Couldn't get URL")
        actual_url = url

    # Lets try to clean the URL up as well by removing all but netloc and path
    url_object = urllib.parse.urlparse(actual_url)
    url = url_object.scheme + "://" + url_object.netloc + url_object.path

    return url

'''Search for an image on google. Grabs the first one, if it ends in html or php
 grab the next one in the array'''
def image_search(search_term):
    search_term = re.sub(r'\W+', '', search_term)
    url = "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + search_term + "&start=0&safe=active"
    response = None
    search_results = []
    try:
        response = requests.get(url).json()
    except:
        response = None

    if response and len(response['responseData']['results']) > 0:
        logging.info("We have %s results, yay!" % len(response['responseData']['results']) )
        for result in response['responseData']['results']:
            image_url = result['unescapedUrl']
            real_url = get_redirect_url(image_url)
            if real_url.endswith('html') or real_url.endswith('php') or real_url.endswith('htm'):
                logging.info("Could not get image url for %s" % real_url)
                continue
            else:
                search_results.append(real_url)
    else:
        logging.info("Couldn't find any results! *shrugs*")

    logging.debug("Returning %s results" % len(search_results))

    return search_results

''' Downloads the given file url to the cache folder'''
def download_file(url):
    file_name = url.split('/')[-1]
    ua = UserAgent()
    # todo create cache folder if it does not already exist.
    cache_folder = 'cache'
    if os.path.isfile(os.path.join(cache_folder, file_name)):
        logging.info('File already exists, skipping download')
        return os.path.join(cache_folder, file_name)
    else:
        logging.info("Downloading %s" % url)
        response = None
        try:
            response = requests.get(url, stream=True, headers = {'User-Agent': ua.chrome } )
        except:
            resposne = None

        fail = False
        try:
            if response.status_code == 200:
                with open(os.path.join('cache', file_name), 'wb') as out_file:
                    for chunk in response.iter_content(1024):
                        out_file.write(chunk)
                    else:
                        logging.critical("Couldn't download %s" % file_name)
        except:
            fail = True
        del response
        if not fail:
            file_path = os.path.join(cache_folder, file_name)
            logging.info("Downloaded %s" % file_path)
            return file_path
        else:
            return None

'''This function should always return the help text'''
def get_help(chat_id):
    return_message = 'Query Options:\n\n'
    return_message = return_message + '/get search for this' + '\n\n'

    return_message = return_message + 'Command Options:\n\n'
    for item in sorted(keyword_map):
        return_message = return_message + '/' + item + '\n'

    send_simple_message(chat_id, return_message)

'''Gets and sends an image based on message text'''
def get_image(chat_id, messagetext):
    message_parts = messagetext.split(' ')
    message_parts.pop(0)
    search_terms = ' '.join(message_parts)
    search_results = image_search(search_terms)
    image_sent = False
    if search_results:
        for image_url in search_results:
            # break out of this loop if an image is successfully sent
            if image_sent:
                logging.info("Already sent an image, skip the rest of the results")
                break

            logging.debug("Supposedly URL: %s" % image_url)
            file_name = download_file(image_url)
            if not file_name:
                logging.critical("%s does not exist" % file_name)
                continue

            logging.debug("file: %s" % file_name)
            # Try to be smart about the content type
            try:
                if file_name.endswith('gif'):
                    send_document(chat_id, file_name)
                    image_sent = True
                elif file_name.endswith('png') or file_name.endswith('jpg') or file_name.endswith('jpeg'):
                    send_photo(chat_id, file_name)
                    image_sent = True
                else:
                    image_sent = False
            except:
                image_sent = False
    else:
        image_sent = False

    return image_sent, search_terms

def get_static(chat_id, messagetext):
    command = messagetext.strip('/')
    if command in keyword_map:
        file_ext = keyword_map[command].split('.')[-1]
        if file_ext in ['jpg', 'jpeg', 'png']:
            file_name = download_file(keyword_map[command])
            logging.info("Sending photo %s" % file_name)
            send_photo(chat_id, file_name)
            return True
        elif file_ext in ['gif']:
            file_name = download_file(keyword_map[command])
            logging.info("Sending gif %s" % file_name)
            send_document(chat_id, file_name)
            return True
        else:
            return_message = get_redirect_url(keyword_map[command])
            send_simple_message(chat_id, return_message)
            return True
    else:
        return None

'''All bot logic happens here and calls out to functions'''
def do_bot_stuff(update_id):
    try:
        data = urllib.parse.urlencode({'offset': format(update_id),
                                       'limit': '100', 'timeout': '60'})

        response = urllib.request.urlopen(bot_url + 'getUpdates',
                                          data.encode('utf-8'))

        reader = codecs.getreader("utf-8")
        data = json.load(reader(response))
    except Exception as e:
        logging.warning('Something went wrong when fetching update: ' + str(e))
        return update_id

    if data['ok'] == True:
        for update in data['result']:

            # take new update id
            update_id = update['update_id'] + 1
            message = update['message']

            # respond if this is a message containing text
            if 'text' in message:
                # Get text of the message
                messagetext = str(message['text'])

                # Skip anything that isn't a slash command
                if not messagetext.startswith('/'):
                    continue

                # Get chat id, which is used to send the return message
                chat_id = message['chat']['id']

                # Show help text
                if messagetext == '/help':
                    get_help(chat_id)
                # get a image via google image search API
                elif messagetext.startswith('/get'):
                    result, term = get_image(chat_id, messagetext)
                    if result:
                        logging.info("Got Image: %s" % result)
                    else:
                        return_message = "Couldn't find result for %s" % term
                        send_simple_message(chat_id, return_message)
                # Is it a statically set item?
                else:
                    result =  get_static(chat_id, messagetext)
                    logging.info("Got Image: %s" % result)
                    continue

    return update_id

'''Main program loop, handles timing and long polling'''
def main():
    # make sure the offset file exists and contains an integer
    try:
        file = open(offset_file, 'rt')
        update_id = int(file.read())
        file.close()
    except Exception as e:
        logging.warning(offset_file + 'does not exist. Creating it now' + str(e))
        with open(offset_file, 'w') as file:
            file.write('0')
            update_id = 0
    # main program loop
    while True:
    # process updates
        newupdate_id = do_bot_stuff(update_id)

        # Write the update ID to a file and sleep 3 seconds if we processed updates
        if (newupdate_id != update_id):
            file = open(offset_file, 'wt')
            file.write(str(newupdate_id))
            file.close()
            time.sleep(3)
        else:
            # Otherwise we can wait some more during long polling if we have to.
            time.sleep(1)

        # use new update ID
        update_id = newupdate_id

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: latin-1; -*-

import os, shutil, platform, subprocess

def chDir(dirname):
    os.chdir(dirname)
    print "[in %s]" % os.getcwd()

def isUnix():
    uname = platform.uname()
    return not (uname[0] == 'Windows' or uname[2] == 'Windows')

def main():
    # update git directories
    if 1:
        for gitdir in ['keygen', 'MetaforSetup', 'parasolid']:
            chDir(gitdir)
            if isUnix():
                cmd='git pull origin master'
            else:
                cmd=r'"C:\Program Files\Git\bin\sh.exe" --login -c "git pull origin master"'
            print cmd
            # os.system necessite des "" en plus autour de la cmd)
            #os.system('"%s"' % cmd)
            status = subprocess.call(cmd, shell=True)
            if status: raise Exception('"%s" FAILED with error %d' % (cmd, status))
            print 'status=', status
            chDir('..')

    #update svn directories
    if 1:
        # set SVN_SSH, sinon: "can't create tunnel"
        os.environ['SVN_SSH']=r'C:\\Program Files\\TortoiseSVN\\bin\\TortoisePlink.exe'  # '\\\\' ou 'r et \\' !!
        for svndir in ['oo_nda', 'oo_meta', 'mtStart']:
            cmd='svn update %s' % svndir
            print cmd
            status = subprocess.call(cmd, shell=True)
            if status: raise Exception('"%s" FAILED with error %d' % (cmd, status))
            #print 'status=', status

    # clean build dir
    if 1:
        print 'removing build dir'
        if os.path.isdir('oo_metaB'):
            shutil.rmtree('oo_metaB')
        os.mkdir('oo_metaB')
        chDir('oo_metaB')

    # make
    if 1:
        if isUnix():
            cmd='cmake -G"Eclipse CDT4 - Unix Makefiles"' \
                ' -D_ECLIPSE_VERSION=4.4' \
                ' -DCMAKE_ECLIPSE_GENERATE_SOURCE_PROJECT=TRUE' \
                ' -C ../oo_meta/CMake/ubuntu.cmake' \
                ' ../oo_meta'
        else:
            cmd=r'cmake -C ..\oo_meta\CMake\garfield.cmake ..\oo_meta'
        os.system(cmd)

    if 1:
        if isUnix():
            os.system('make -j 12')
        else:
            os.system('BuildConsole Metafor.sln /rebuild /cfg="Release|x64"')

if __name__=="__main__":
    main()

<commit_msg>correction linux<commit_after>#! /usr/bin/env python
# -*- coding: latin-1; -*-

import os, shutil, platform, subprocess

def chDir(dirname):
    os.chdir(dirname)
    print "[in %s]" % os.getcwd()

def isUnix():
    uname = platform.uname()
    return not (uname[0] == 'Windows' or uname[2] == 'Windows')

def main():
    # update git directories
    if 1:
        for gitdir in ['keygen', 'MetaforSetup', 'parasolid']:
            chDir(gitdir)
            if isUnix():
                cmd='git pull origin master'
            else:
                cmd=r'"C:\Program Files\Git\bin\sh.exe" --login -c "git pull origin master"'
            print cmd
            # os.system necessite des "" en plus autour de la cmd)
            #os.system('"%s"' % cmd)
            status = subprocess.call(cmd, shell=True)
            if status: raise Exception('"%s" FAILED with error %d' % (cmd, status))
            print 'status=', status
            chDir('..')

    #update svn directories
    if 1:
        # set SVN_SSH, sinon: "can't create tunnel"
        if not isUnix():
            os.environ['SVN_SSH']=r'C:\\Program Files\\TortoiseSVN\\bin\\TortoisePlink.exe'  # '\\\\' ou 'r et \\' !!
        for svndir in ['oo_nda', 'oo_meta', 'mtStart']:
            cmd='svn update %s' % svndir
            print cmd
            status = subprocess.call(cmd, shell=True)
            if status: raise Exception('"%s" FAILED with error %d' % (cmd, status))
            #print 'status=', status

    # clean build dir
    if 1:
        print 'removing build dir'
        if os.path.isdir('oo_metaB'):
            shutil.rmtree('oo_metaB')
        os.mkdir('oo_metaB')
        chDir('oo_metaB')

    # make
    if 1:
        if isUnix():
            cmd='cmake -G"Eclipse CDT4 - Unix Makefiles"' \
                ' -D_ECLIPSE_VERSION=4.4' \
                ' -DCMAKE_ECLIPSE_GENERATE_SOURCE_PROJECT=TRUE' \
                ' -C ../oo_meta/CMake/ubuntu.cmake' \
                ' ../oo_meta'
        else:
            cmd=r'cmake -C ..\oo_meta\CMake\garfield.cmake ..\oo_meta'
        os.system(cmd)

    if 1:
        if isUnix():
            os.system('make -j 12')
        else:
            os.system('BuildConsole Metafor.sln /rebuild /cfg="Release|x64"')

if __name__=="__main__":
    main()

<|endoftext|>


<commit_before>__version__ = "2.12.3"
<commit_msg>bump version<commit_after>__version__ = "2.13.0"
<|endoftext|>


<commit_before>#!/usr/bin/python
"""better_twill is a small wrapper around twill to set some sane defaults and
monkey-patch some better versions of some of twill's methods.
It also handles twill's absense.
"""

import os
from os.path import abspath, dirname, join
import sys
from pkg_resources import parse_version as pv
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

# On OSX lxml needs to be imported before twill to avoid Resolver issues
# somehow caused by the mac specific 'ic' module
try:
    from lxml import etree
except ImportError:
    pass

try:
    import twill
except ImportError:
    twill = None

# When twill tries to connect to a site before the site is up, it raises an
# exception.  In 0.9b1, it's urlib2.URLError, but in -latest, it's
# _mechanize_dist._mechanize.BrowserStateError.
try:
    from _mechanize_dist._mechanize import BrowserStateError as ConnectError
except ImportError:
    from urllib2 import URLError as ConnectError

if twill:
    # We want Trac to generate valid html, and therefore want to test against
    # the html as generated by Trac.  "tidy" tries to clean up broken html, and
    # is responsible for one difficult to track down testcase failure (for
    # #5497).  Therefore we turn it off here.
    twill.commands.config('use_tidy', '0')

    # setup short names to reduce typing
    # This twill browser (and the tc commands that use it) are essentially
    # global, and not tied to our test fixture.
    tc = twill.commands
    b = twill.get_browser()

    # Setup XHTML validation for all retrieved pages
    try:
        from lxml import etree
    except ImportError:
        print "SKIP: validation of XHTML output in functional tests " \
              "(no lxml installed)"
        etree = None

    if etree and pv(etree.__version__) < pv('2.0.0'):
        # 2.0.7 and 2.1.x are known to work.
        print "SKIP: validation of XHTML output in functional tests " \
              "(lxml < 2.0, api incompatibility)"
        etree = None

    if etree:
        class _Resolver(etree.Resolver):
            base_dir = dirname(abspath(__file__))

            def resolve(self, system_url, public_id, context):
                return self.resolve_filename(join(self.base_dir,
                                                  system_url.split("/")[-1]),
                                             context)

        _parser = etree.XMLParser(dtd_validation=True)
        _parser.resolvers.add(_Resolver())
        etree.set_default_parser(_parser)

        def _format_error_log(data, log):
            msg = []
            for entry in log:
                context = data.splitlines()[max(0, entry.line - 5):
                                            entry.line + 6]
                msg.append("\n# %s\n# URL: %s\n# Line %d, column %d\n\n%s\n"
                    % (entry.message, entry.filename,
                       entry.line, entry.column,
                       "\n".join([each.decode('utf-8') for each in context])))
            return "\n".join(msg).encode('ascii', 'xmlcharrefreplace')

        def _validate_xhtml(func_name, *args, **kwargs):
            page = b.get_html()
            if "xhtml1-strict.dtd" not in page:
                return
            etree.clear_error_log()
            try:
                doc = etree.parse(StringIO(page), base_url=b.get_url())
            except etree.XMLSyntaxError, e:
                raise twill.errors.TwillAssertionError(
                    _format_error_log(page, e.error_log))

        b._post_load_hooks.append(_validate_xhtml)

    # When we can't find something we expected, or find something we didn't
    # expect, it helps the debugging effort to have a copy of the html to
    # analyze.
    def twill_write_html():
        """Write the current html to a file.  Name the file based on the
        current testcase.
        """
        frame = sys._getframe()
        while frame:
            if frame.f_code.co_name in ('runTest', 'setUp', 'tearDown'):
                testcase = frame.f_locals['self']
                testname = testcase.__class__.__name__
                tracdir = testcase._testenv.tracdir
                break
            frame = frame.f_back
        else:
            # We didn't find a testcase in the stack, so we have no clue what's
            # going on.
            raise Exception("No testcase was found on the stack.  This was "
                "really not expected, and I don't know how to handle it.")

        filename = os.path.join(tracdir, 'log', "%s.html" % testname)
        html_file = open(filename, 'w')
        html_file.write(b.get_html())
        html_file.close()

        return filename

    # Twill isn't as helpful with errors as I'd like it to be, so we replace
    # the formvalue function.  This would be better done as a patch to Twill.
    def better_formvalue(form, field, value, fv=tc.formvalue):
        try:
            fv(form, field, value)
        except (twill.errors.TwillAssertionError,
                twill.utils.ClientForm.ItemNotFoundError), e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.formvalue = better_formvalue

    # Twill's formfile function leaves a filehandle open which prevents the
    # file from being deleted on Windows.  Since we would just assume use a
    # StringIO object in the first place, allow the file-like object to be
    # provided directly.
    def better_formfile(formname, fieldname, filename, content_type=None,
                        fp=None):
        if not fp:
            filename = filename.replace('/', os.path.sep)
            temp_fp = open(filename, 'rb')
            data = temp_fp.read()
            temp_fp.close()
            fp = StringIO(data)

        form = b.get_form(formname)
        control = b.get_form_field(form, fieldname)

        if not control.is_of_kind('file'):
            raise twill.errors.TwillException('ERROR: field is not a file '
                                              'upload field!')

        b.clicked(form, control)
        control.add_file(fp, content_type, filename)
    tc.formfile = better_formfile

    # Twill's tc.find() does not provide any guidance on what we got instead of
    # what was expected.
    def better_find(what, flags='', tcfind=tc.find):
        try:
            tcfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.find = better_find
    def better_notfind(what, flags='', tcnotfind=tc.notfind):
        try:
            tcnotfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.notfind = better_notfind
else:
    b = tc = None
<commit_msg>0.12dev: Wrap the global twill browser object in a transparent proxy that accesses the object via `twill.get_browser()`, as the object can change after a `browser_reset()` call.<commit_after>#!/usr/bin/python
"""better_twill is a small wrapper around twill to set some sane defaults and
monkey-patch some better versions of some of twill's methods.
It also handles twill's absense.
"""

import os
from os.path import abspath, dirname, join
import sys
from pkg_resources import parse_version as pv
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

# On OSX lxml needs to be imported before twill to avoid Resolver issues
# somehow caused by the mac specific 'ic' module
try:
    from lxml import etree
except ImportError:
    pass

try:
    import twill
except ImportError:
    twill = None

# When twill tries to connect to a site before the site is up, it raises an
# exception.  In 0.9b1, it's urlib2.URLError, but in -latest, it's
# _mechanize_dist._mechanize.BrowserStateError.
try:
    from _mechanize_dist._mechanize import BrowserStateError as ConnectError
except ImportError:
    from urllib2 import URLError as ConnectError

if twill:
    # We want Trac to generate valid html, and therefore want to test against
    # the html as generated by Trac.  "tidy" tries to clean up broken html, and
    # is responsible for one difficult to track down testcase failure (for
    # #5497).  Therefore we turn it off here.
    twill.commands.config('use_tidy', '0')

    # We use a transparent proxy to access the global browser object through
    # twill.get_browser(), as the browser can be destroyed by browser_reset()
    # (see #7472).
    class _BrowserProxy(object):
        def __getattribute__(self, name):
            return getattr(twill.get_browser(), name)

        def __setattr(self, name, value):
            setattr(twill.get_browser(), name, value)

    # setup short names to reduce typing
    # This twill browser (and the tc commands that use it) are essentially
    # global, and not tied to our test fixture.
    tc = twill.commands
    b = _BrowserProxy()

    # Setup XHTML validation for all retrieved pages
    try:
        from lxml import etree
    except ImportError:
        print "SKIP: validation of XHTML output in functional tests " \
              "(no lxml installed)"
        etree = None

    if etree and pv(etree.__version__) < pv('2.0.0'):
        # 2.0.7 and 2.1.x are known to work.
        print "SKIP: validation of XHTML output in functional tests " \
              "(lxml < 2.0, api incompatibility)"
        etree = None

    if etree:
        class _Resolver(etree.Resolver):
            base_dir = dirname(abspath(__file__))

            def resolve(self, system_url, public_id, context):
                return self.resolve_filename(join(self.base_dir,
                                                  system_url.split("/")[-1]),
                                             context)

        _parser = etree.XMLParser(dtd_validation=True)
        _parser.resolvers.add(_Resolver())
        etree.set_default_parser(_parser)

        def _format_error_log(data, log):
            msg = []
            for entry in log:
                context = data.splitlines()[max(0, entry.line - 5):
                                            entry.line + 6]
                msg.append("\n# %s\n# URL: %s\n# Line %d, column %d\n\n%s\n"
                    % (entry.message, entry.filename,
                       entry.line, entry.column,
                       "\n".join([each.decode('utf-8') for each in context])))
            return "\n".join(msg).encode('ascii', 'xmlcharrefreplace')

        def _validate_xhtml(func_name, *args, **kwargs):
            page = b.get_html()
            if "xhtml1-strict.dtd" not in page:
                return
            etree.clear_error_log()
            try:
                doc = etree.parse(StringIO(page), base_url=b.get_url())
            except etree.XMLSyntaxError, e:
                raise twill.errors.TwillAssertionError(
                    _format_error_log(page, e.error_log))

        b._post_load_hooks.append(_validate_xhtml)

    # When we can't find something we expected, or find something we didn't
    # expect, it helps the debugging effort to have a copy of the html to
    # analyze.
    def twill_write_html():
        """Write the current html to a file.  Name the file based on the
        current testcase.
        """
        frame = sys._getframe()
        while frame:
            if frame.f_code.co_name in ('runTest', 'setUp', 'tearDown'):
                testcase = frame.f_locals['self']
                testname = testcase.__class__.__name__
                tracdir = testcase._testenv.tracdir
                break
            frame = frame.f_back
        else:
            # We didn't find a testcase in the stack, so we have no clue what's
            # going on.
            raise Exception("No testcase was found on the stack.  This was "
                "really not expected, and I don't know how to handle it.")

        filename = os.path.join(tracdir, 'log', "%s.html" % testname)
        html_file = open(filename, 'w')
        html_file.write(b.get_html())
        html_file.close()

        return filename

    # Twill isn't as helpful with errors as I'd like it to be, so we replace
    # the formvalue function.  This would be better done as a patch to Twill.
    def better_formvalue(form, field, value, fv=tc.formvalue):
        try:
            fv(form, field, value)
        except (twill.errors.TwillAssertionError,
                twill.utils.ClientForm.ItemNotFoundError), e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.formvalue = better_formvalue

    # Twill's formfile function leaves a filehandle open which prevents the
    # file from being deleted on Windows.  Since we would just assume use a
    # StringIO object in the first place, allow the file-like object to be
    # provided directly.
    def better_formfile(formname, fieldname, filename, content_type=None,
                        fp=None):
        if not fp:
            filename = filename.replace('/', os.path.sep)
            temp_fp = open(filename, 'rb')
            data = temp_fp.read()
            temp_fp.close()
            fp = StringIO(data)

        form = b.get_form(formname)
        control = b.get_form_field(form, fieldname)

        if not control.is_of_kind('file'):
            raise twill.errors.TwillException('ERROR: field is not a file '
                                              'upload field!')

        b.clicked(form, control)
        control.add_file(fp, content_type, filename)
    tc.formfile = better_formfile

    # Twill's tc.find() does not provide any guidance on what we got instead of
    # what was expected.
    def better_find(what, flags='', tcfind=tc.find):
        try:
            tcfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.find = better_find
    def better_notfind(what, flags='', tcnotfind=tc.notfind):
        try:
            tcnotfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.notfind = better_notfind
else:
    b = tc = None
<|endoftext|>


<commit_before>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils.tests')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<commit_msg>actually run the algorithms tests<commit_after>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__    = "$LastChangedDate$"
__url__     = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

from neuroimaging import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms',
  'neuroimaging.algorithms.tests',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.modalities.fmri',
  'neuroimaging.modalities.fmri.tests',
  'neuroimaging.modalities.fmri.fmristat',
  'neuroimaging.modalities.fmri.fmristat.tests',
  'neuroimaging.utils.tests')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr):
            return False
    return True

def haslength(obj):
    return hasattr(obj,"__len__")

def flatten(arr, dim=0):
    if len(arr.shape) < 2:
        return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

def reorder(seq, order):
    return [seq[i] for i in order]

def reverse(seq):
    return reorder(seq, range(len(seq)-1, -1, -1))

def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try:
        return getattr(module, objectname)
    except AttributeError:
        return None

<|endoftext|>