<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Functions to setup xvfb, which is used by the linux machines.
"""

import os
import platform
import signal
import subprocess
import tempfile
import time

def _XvfbPidFilename(slave_build_name):
  """Returns the filename to the Xvfb pid file.  This name is unique for each
  builder. This is used by the linux builders."""
  return os.path.join(tempfile.gettempdir(),
                      'xvfb-' + slave_build_name  + '.pid')

def StartVirtualX(slave_build_name, build_dir, with_wm=True, server_dir=None):
  """Start a virtual X server and set the DISPLAY environment variable so sub
  processes will use the virtual X server.  Also start icewm. This only works
  on Linux and assumes that xvfb and icewm are installed.

  Args:
    slave_build_name: The name of the build that we use for the pid file.
        E.g., webkit-rel-linux.
    build_dir: The directory where binaries are produced.  If this is non-empty,
        we try running xdisplaycheck from |build_dir| to verify our X
        connection.
    with_wm: Whether we add a window manager to the display too.
    server_dir: Directory to search for the server executable.
  """
  # We use a pid file to make sure we don't have any xvfb processes running
  # from a previous test run.
  StopVirtualX(slave_build_name)

  # Figure out which X server to try.
  cmd = 'Xvfb'
  if server_dir and os.path.exists(server_dir):
    cmd = os.path.join(server_dir, 'Xvfb.' + platform.architecture()[0])
    if not os.path.exists(cmd):
      cmd = os.path.join(server_dir, 'Xvfb')
    if not os.path.exists(cmd):
      print 'No Xvfb found in designated server path:', server_dir
      raise Exception('No virtual server')

  # Start a virtual X server that we run the tests in.  This makes it so we can
  # run the tests even if we didn't start the tests from an X session.
  proc = subprocess.Popen([cmd, ':9', '-screen', '0', '1024x768x24', '-ac'],
                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  xvfb_pid_filename = _XvfbPidFilename(slave_build_name)
  open(xvfb_pid_filename, 'w').write(str(proc.pid))
  os.environ['DISPLAY'] = ':9'

  # Verify that Xvfb has started by using xdisplaycheck.
  if build_dir:
    xdisplaycheck_path = os.path.join(build_dir, 'xdisplaycheck')
    if os.path.exists(xdisplaycheck_path):
      print 'Verifying Xvfb has started...'
      checkstarttime = time.time()
      xdisplayproc = subprocess.Popen([xdisplaycheck_path],
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.STDOUT)
      # Wait for xdisplaycheck to exit.
      logs = xdisplayproc.communicate()[0]
      checktime = time.time() - checkstarttime
      if xdisplayproc.returncode != 0:
        print 'xdisplaycheck failed after %d seconds.' % checktime
        print 'xdisplaycheck output:'
        for l in logs.splitlines():
          print '> %s' % l
        rc = proc.poll()
        if rc is None:
          print 'Xvfb still running, stopping.'
          proc.terminate()
        else:
          print 'Xvfb exited, code %d' % rc

        print 'Xvfb output:'
        for l in proc.communicate()[0].splitlines():
          print '> %s' % l
        raise Exception(logs)
      else:
        print 'xdisplaycheck succeeded after %d seconds.' % checktime
        print 'xdisplaycheck output:'
        for l in logs.splitlines():
          print '> %s' % l
      print '...OK'

  if with_wm:
    # Some ChromeOS tests need a window manager.
    subprocess.Popen('icewm', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    print 'Window manager (icewm) started.'
  else:
    print 'No window manager required.'

def StopVirtualX(slave_build_name):
  """Try and stop the virtual X server if one was started with StartVirtualX.
  When the X server dies, it takes down the window manager with it.
  If a virtual x server is not running, this method does nothing."""
  xvfb_pid_filename = _XvfbPidFilename(slave_build_name)
  if os.path.exists(xvfb_pid_filename):
    xvfb_pid = int(open(xvfb_pid_filename).read())
    print 'Stopping Xvfb with pid %d ...' % xvfb_pid
    # If the process doesn't exist, we raise an exception that we can ignore.
    try:
      os.kill(xvfb_pid, signal.SIGKILL)
    except OSError:
      print '... killing failed, presuming unnecessary.'
    os.remove(xvfb_pid_filename)
    print 'Xvfb pid file removed'
<commit_msg>Make sure previous displays are stopped.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Functions to setup xvfb, which is used by the linux machines.
"""

import os
import platform
import signal
import subprocess
import tempfile
import time

def _XvfbDisplayIndex(slave_build_name):
  return '9'

def _XvfbPidFilename(slave_build_name):
  """Returns the filename to the Xvfb pid file.  This name is unique for each
  builder. This is used by the linux builders."""
  return os.path.join(tempfile.gettempdir(),
                      'xvfb-' + _XvfbDisplayIndex(slave_build_name)  + '.pid')

def StartVirtualX(slave_build_name, build_dir, with_wm=True, server_dir=None):
  """Start a virtual X server and set the DISPLAY environment variable so sub
  processes will use the virtual X server.  Also start icewm. This only works
  on Linux and assumes that xvfb and icewm are installed.

  Args:
    slave_build_name: The name of the build that we use for the pid file.
        E.g., webkit-rel-linux.
    build_dir: The directory where binaries are produced.  If this is non-empty,
        we try running xdisplaycheck from |build_dir| to verify our X
        connection.
    with_wm: Whether we add a window manager to the display too.
    server_dir: Directory to search for the server executable.
  """
  # We use a pid file to make sure we don't have any xvfb processes running
  # from a previous test run.
  StopVirtualX(slave_build_name)

  xdisplaycheck_path = None
  if build_dir:
    xdisplaycheck_path = os.path.join(build_dir, 'xdisplaycheck')

  display = ':%s' % _XvfbDisplayIndex(slave_build_name)
  # Note we don't add the optional screen here (+ '.0')
  os.environ['DISPLAY'] = display

  if xdisplaycheck_path and os.path.exists(xdisplaycheck_path):
    print 'Verifying Xvfb is not running ...'
    checkstarttime = time.time()
    xdisplayproc = subprocess.Popen([xdisplaycheck_path, '--noserver'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
    # Wait for xdisplaycheck to exit.
    logs = xdisplayproc.communicate()[0]
    if xdisplayproc.returncode == 0:
      print 'xdisplaycheck says there is a display still running, exiting...'
      raise Exception('Display already present.')

  # Figure out which X server to try.
  cmd = 'Xvfb'
  if server_dir and os.path.exists(server_dir):
    cmd = os.path.join(server_dir, 'Xvfb.' + platform.architecture()[0])
    if not os.path.exists(cmd):
      cmd = os.path.join(server_dir, 'Xvfb')
    if not os.path.exists(cmd):
      print 'No Xvfb found in designated server path:', server_dir
      raise Exception('No virtual server')

  # Start a virtual X server that we run the tests in.  This makes it so we can
  # run the tests even if we didn't start the tests from an X session.
  proc = subprocess.Popen([cmd, display, '-screen', '0', '1024x768x24', '-ac'],
                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  xvfb_pid_filename = _XvfbPidFilename(slave_build_name)
  open(xvfb_pid_filename, 'w').write(str(proc.pid))

  # Verify that Xvfb has started by using xdisplaycheck.
  if xdisplaycheck_path and os.path.exists(xdisplaycheck_path):
    print 'Verifying Xvfb has started...'
    checkstarttime = time.time()
    xdisplayproc = subprocess.Popen([xdisplaycheck_path],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
    # Wait for xdisplaycheck to exit.
    logs = xdisplayproc.communicate()[0]
    checktime = time.time() - checkstarttime
    if xdisplayproc.returncode != 0:
      print 'xdisplaycheck failed after %d seconds.' % checktime
      print 'xdisplaycheck output:'
      for l in logs.splitlines():
        print '> %s' % l
      rc = proc.poll()
      if rc is None:
        print 'Xvfb still running, stopping.'
        proc.terminate()
      else:
        print 'Xvfb exited, code %d' % rc

      print 'Xvfb output:'
      for l in proc.communicate()[0].splitlines():
        print '> %s' % l
      raise Exception(logs)
    else:
      print 'xdisplaycheck succeeded after %d seconds.' % checktime
      print 'xdisplaycheck output:'
      for l in logs.splitlines():
        print '> %s' % l
    print '...OK'

  if with_wm:
    # Some ChromeOS tests need a window manager.
    subprocess.Popen('icewm', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    print 'Window manager (icewm) started.'
  else:
    print 'No window manager required.'

def StopVirtualX(slave_build_name):
  """Try and stop the virtual X server if one was started with StartVirtualX.
  When the X server dies, it takes down the window manager with it.
  If a virtual x server is not running, this method does nothing."""
  xvfb_pid_filename = _XvfbPidFilename(slave_build_name)
  if os.path.exists(xvfb_pid_filename):
    xvfb_pid = int(open(xvfb_pid_filename).read())
    print 'Stopping Xvfb with pid %d ...' % xvfb_pid
    # If the process doesn't exist, we raise an exception that we can ignore.
    try:
      os.kill(xvfb_pid, signal.SIGKILL)
    except OSError:
      print '... killing failed, presuming unnecessary.'
    os.remove(xvfb_pid_filename)
    print 'Xvfb pid file removed'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import os

from datetime import datetime

# Config
dbs = ["F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD", "F_ST_ETIENNE_SESSAD_TED"]
tables = ["actes",
"actes_non_factures",
"adresses",
"annexes",
"autorite_parentale",
"avs_genre",
"b2",
"banques",
"blocage",
"caisses",
"categorie_code_nf",
"civilite",
"classes",
"code_events",
"code_nf",
"codes_gestion",
"conge",
"conges_motifs",
"contact_qui",
"contacts",
"contrat",
"convocations",
"cp",
"crm_medicaux",
"csp",
"decisions",
"departements",
"details_actes",
"details_ev",
"details_factures",
"details_rr",
"details_rs",
"diag_exter",
"discipline",
"dossiers",
"dossiers_ecoles",
"dossiers_mises",
"droits",
"ecoles",
"etablissement",
"ev",
"factures",
"fin_attente_motifs",
"fin_bilan",
"fratrie",
"genre_etab",
"genre_pc",
"grands_regimes",
"historique",
"ins_motif_exprim",
"ins_motifs",
"ins_provenance",
"intervenants",
"intervenants_etablissements",
"intervenants_exterieurs",
"maj",
"marque",
"medecins_exterieurs",
"messages",
"mises",
"nationalite",
"noemie",
"notes",
"parametres",
"parente",
"pc",
"pds",
"periodes_pc",
"pmt",
"prix_journee",
"profession",
"quartiers",
"rang",
"reglements",
"reguls",
"rejets_motifs",
"rm",
"rr",
"rs",
"save_data",
"section_educative",
"sexe",
"situation_familiale",
"sollicitation",
"sor_motifs",
"sor_orientation",
"stats",
"suivi",
"taches",
"transmission",
"transparams",
"transports",
"type_acte",
"type_acte_categorie",
"type_transport",
"users"]

export_path = os.path.join("C:\\", "export", datetime.now().strftime("%Y%m%d-%H%M%S"))

os.mkdir(export_path)

for db in dbs:
    db_dir = os.path.join(export_path, db)
    os.mkdir(db_dir)
    for table in tables:
        os.system('sqlcmd -S SRVAPS -E -h-1 -s, -Q"SET NOCOUNT ON;' + \
                'DECLARE @colnames VARCHAR(max); ' + \
                "SELECT @colnames = COALESCE(@colnames + ',', '') " + \
                '+ column_name from %s.INFORMATION_SCHEMA.COLUMNS ' % db + \
                " where TABLE_NAME='%s'; select @colnames;\" " % table + \
                '-o "%s-title.csv"' % os.path.join(db_dir, table))
        os.system('bcp %s.dbo.%s out %s-data.csv -w  -t"\\",\\"" -T -SSRVAPS' % (db, table, os.path.join(db_dir, table)))
        title = open("%s-title.csv" % os.path.join(db_dir, table), 'r')
        data = open("%s-data.csv" % os.path.join(db_dir, table), 'r')
	res = open("%s.csv" % os.path.join(db_dir, table), "a+")
        title_content = re.sub(r'\s+$', '\n', title.readline())
        res.write(title_content)
        lines = data.readlines()
	for i, line in enumerate(lines):
	    line = unicode(line, 'cp1252')
	    line = line.replace('\0', '').replace('\r', '')
	    if i == 0:
		line = re.sub('^', '"', line[2:])
	    else:
                line = re.sub('^', '"', line)
	    if line and line != "\n":
                line = re.sub('\n', '"\n', line)
	    if line != '"' and line != '"\n' and line != '""\n':
                res.write(line.encode('utf-8'))
        title.close()
        data.close()
        res.close()
        os.remove("%s-title.csv" % os.path.join(db_dir, table))
        os.remove("%s-data.csv" % os.path.join(db_dir, table))

<commit_msg>export db: fix columns export<commit_after>#!/usr/bin/env python

import re
import os

from datetime import datetime

# Config
dbs = ["F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD", "F_ST_ETIENNE_SESSAD_TED"]
tables = ["actes",
"actes_non_factures",
"adresses",
"annexes",
"autorite_parentale",
"avs_genre",
"b2",
"banques",
"blocage",
"caisses",
"categorie_code_nf",
"civilite",
"classes",
"code_events",
"code_nf",
"codes_gestion",
"conge",
"conges_motifs",
"contact_qui",
"contacts",
"contrat",
"convocations",
"cp",
"crm_medicaux",
"csp",
"decisions",
"departements",
"details_actes",
"details_ev",
"details_factures",
"details_rr",
"details_rs",
"diag_exter",
"discipline",
"dossiers",
"dossiers_ecoles",
"dossiers_mises",
"droits",
"ecoles",
"etablissement",
"ev",
"factures",
"fin_attente_motifs",
"fin_bilan",
"fratrie",
"genre_etab",
"genre_pc",
"grands_regimes",
"historique",
"ins_motif_exprim",
"ins_motifs",
"ins_provenance",
"intervenants",
"intervenants_etablissements",
"intervenants_exterieurs",
"maj",
"marque",
"medecins_exterieurs",
"messages",
"mises",
"nationalite",
"noemie",
"notes",
"parametres",
"parente",
"pc",
"pds",
"periodes_pc",
"pmt",
"prix_journee",
"profession",
"quartiers",
"rang",
"reglements",
"reguls",
"rejets_motifs",
"rm",
"rr",
"rs",
"save_data",
"section_educative",
"sexe",
"situation_familiale",
"sollicitation",
"sor_motifs",
"sor_orientation",
"stats",
"suivi",
"taches",
"transmission",
"transparams",
"transports",
"type_acte",
"type_acte_categorie",
"type_transport",
"users"]

export_path = os.path.join("C:\\", "export", datetime.now().strftime("%Y%m%d-%H%M%S"))

os.mkdir(export_path)

for db in dbs:
    db_dir = os.path.join(export_path, db)
    os.mkdir(db_dir)
    for table in tables:
        os.system('sqlcmd -S SRVAPS -E -h-1 -s, -Q"SET NOCOUNT ON;' + \
                ' SELECT column_name from ' + \
                '%s.INFORMATION_SCHEMA.COLUMNS ' % db + \
                " where TABLE_NAME='%s';\" " % table + \
                ' -o "%s-title.csv"' % os.path.join(db_dir, table))
        os.system('bcp %s.dbo.%s out %s-data.csv -w  -t"\\",\\"" -T -SSRVAPS' % (db, table, os.path.join(db_dir, table)))
        title = open("%s-title.csv" % os.path.join(db_dir, table), 'r')
        data = open("%s-data.csv" % os.path.join(db_dir, table), 'r')
        res = open("%s.csv" % os.path.join(db_dir, table), "a+")
        title_content = title.read()
        title_content, nb = re.subn(r'\s+\n', ',', title_content)
        title_content = title_content[:-1] + '\n'
        res.write(title_content)
        lines = data.readlines()
        for i, line in enumerate(lines):
            line = unicode(line, 'cp1252')
            line = line.replace('\0', '').replace('\r', '')
            if i == 0:
                line = re.sub('^', '"', line[2:])
            else:
                line = re.sub('^', '"', line)
            if line and line != "\n":
                line = re.sub('\n', '"\n', line)
            if line != '"' and line != '"\n' and line != '""\n':
                res.write(line.encode('utf-8'))
        title.close()
        data.close()
        res.close()
        os.remove("%s-title.csv" % os.path.join(db_dir, table))
        os.remove("%s-data.csv" % os.path.join(db_dir, table))

<|endoftext|>


<commit_before>import os
import argparse

import numpy as np
import scipy.ndimage as nd
import PIL.Image

from jicbioimage.core.image import Image

from utils import get_microscopy_collection

HERE = os.path.dirname(__file__)
UNPACK = os.path.join(HERE, '..', 'data', 'unpack')
OUTPUT = os.path.join(HERE, '..', 'output')

def generate_surface_from_stack(stack, sd=(2, 2, 2), surface_blur_sd=5):
    """Return a 2D image encoding a height map, generated from the input stack.
    The image is generated by first blurring the stack, then taking the z index
    of the brightest point for each X, Y location. The resultant surface is
    then smoothed with a gaussian filter.

    sd: standard deviation in each direction
    surface_blur_sd: standard deviation of smoothing applied to 2D surface."""

    smoothed_stack = nd.gaussian_filter(stack, sd)
    raw_surface = np.argmax(smoothed_stack, 2)
    smoothed_surface = nd.gaussian_filter(raw_surface, surface_blur_sd)

    return smoothed_surface

def projection_from_stack_and_surface(stack, surface, z_above=1, z_below=1):
    """Return a 2D projection of a 3D stack. The projection is obtained by
    using the elements of the 2D array surface as the Z index for each
    point in the plane."""

    projection = np.zeros(surface.shape, dtype=np.uint8)

    xdim, ydim, zdim = stack.shape
    for x in range(xdim):
        for y in range(ydim):
            z_index = surface[x, y]
            z_min = max(0, z_index-z_above)
            z_max = min(zdim-1, z_index+z_below)
            value = np.mean(stack[x, y, z_min:z_max])
            projection[x, y] = value

    return projection.view(Image)

def save_image(filename, image):
    """Save the given image to a file."""

    with open(filename, 'wb') as f:
        f.write(image.png())

def generate_projections_from_microscope_image(input_file,
                                               wall_channel,
                                               marker_channel):
    """Generate and save projections generated from a stack loaded from the
    input microscopy file. These use a surface extracted from channel 2
    (assumed to be the cell wall channel) which is then used to determine
    projections of that channel, as well as the marker channel (assumed to
    be channel 0)."""

    cell_wall_channel = wall_channel
    marker_channel = marker_channel
    collection = get_microscopy_collection(input_file)

    cell_wall_stack = collection.zstack_array(s=0, c=cell_wall_channel)
    marker_stack = collection.zstack_array(s=0, c=marker_channel)

    surface = generate_surface_from_stack(cell_wall_stack)

    cell_wall_projection = projection_from_stack_and_surface(cell_wall_stack,
                                                             surface, 5, 5)
    marker_projection = projection_from_stack_and_surface(marker_stack,
                                                          surface, 5, 5)

    save_image("wall.png", cell_wall_projection)
    save_image("marker.png", marker_projection)
    pil_im = PIL.Image.fromarray(surface.astype(np.uint8))
    pil_im.save('surface.png')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('input_file', help="Input microscope file.")
    parser.add_argument("-w", "--wall-channel",
                        default=1, type=int,
                        help="Wall channel (zero indexed)")
    parser.add_argument("-m", "--marker-channel",
                        default=0, type=int,
                        help="Marker channel (zero indexed)")

    args = parser.parse_args()

    generate_projections_from_microscope_image(args.input_file,
                                               args.wall_channel,
                                               args.marker_channel)

if __name__ == "__main__":
    main()
<commit_msg>Now dealing with issue of cells touching top layer using padding<commit_after>import os
import argparse

import numpy as np
import scipy.ndimage as nd
import PIL.Image

from jicbioimage.core.image import Image

from utils import get_microscopy_collection

HERE = os.path.dirname(__file__)
UNPACK = os.path.join(HERE, '..', 'data', 'unpack')
OUTPUT = os.path.join(HERE, '..', 'output')

def generate_surface_from_stack(stack, sd=(10, 10, 10), surface_blur_sd=5):
    """Return a 2D image encoding a height map, generated from the input stack.
    The image is generated by first blurring the stack, then taking the z index
    of the brightest point for each X, Y location. The resultant surface is
    then smoothed with a gaussian filter.

    sd: standard deviation in each direction
    surface_blur_sd: standard deviation of smoothing applied to 2D surface."""

    ydim, xdim, zdim = stack.shape
    pad_val = 5
    padding = pad_val * 2
    padded_stack = np.zeros((ydim + padding, xdim + padding, zdim + padding),
                            dtype=stack.dtype)
    start = pad_val
    end_add = pad_val
    padded_stack[start:ydim+end_add, start:xdim+end_add, start:zdim+end_add] = stack
    smoothed_stack = nd.gaussian_filter(padded_stack, sd)
    cropped_stack = smoothed_stack[start:ydim+end_add, start:xdim+end_add, start:zdim+end_add]
    raw_surface = np.argmax(cropped_stack, 2)
    raw_surface[np.logical_and(raw_surface==0, cropped_stack[:,:,0] == 0)] = zdim-1
    smoothed_surface = nd.gaussian_filter(raw_surface, surface_blur_sd)

    return smoothed_surface

def projection_from_stack_and_surface(stack, surface, z_above=1, z_below=1):
    """Return a 2D projection of a 3D stack. The projection is obtained by
    using the elements of the 2D array surface as the Z index for each
    point in the plane."""

    projection = np.zeros(surface.shape, dtype=np.uint8)

    xdim, ydim, zdim = stack.shape
    for x in range(xdim):
        for y in range(ydim):
            z_index = surface[x, y]
            z_min = max(0, z_index-z_above)
            z_max = min(zdim-1, z_index+z_below)
            value = np.mean(stack[x, y, z_min:z_max])
            projection[x, y] = value

    return projection.view(Image)

def save_image(filename, image):
    """Save the given image to a file."""

    with open(filename, 'wb') as f:
        f.write(image.png())

def generate_projections_from_microscope_image(input_file,
                                               wall_channel,
                                               marker_channel):
    """Generate and save projections generated from a stack loaded from the
    input microscopy file. These use a surface extracted from channel 2
    (assumed to be the cell wall channel) which is then used to determine
    projections of that channel, as well as the marker channel (assumed to
    be channel 0)."""

    cell_wall_channel = wall_channel
    marker_channel = marker_channel
    collection = get_microscopy_collection(input_file)

    cell_wall_stack = collection.zstack_array(s=0, c=cell_wall_channel)
    marker_stack = collection.zstack_array(s=0, c=marker_channel)

    surface = generate_surface_from_stack(cell_wall_stack)

    cell_wall_projection = projection_from_stack_and_surface(cell_wall_stack,
                                                             surface, 5, 5)
    marker_projection = projection_from_stack_and_surface(marker_stack,
                                                          surface, 5, 5)

    save_image("wall.png", cell_wall_projection)
    save_image("marker.png", marker_projection)
    pil_im = PIL.Image.fromarray(surface.astype(np.uint8))
    pil_im.save('surface.png')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('input_file', help="Input microscope file.")
    parser.add_argument("-w", "--wall-channel",
                        default=1, type=int,
                        help="Wall channel (zero indexed)")
    parser.add_argument("-m", "--marker-channel",
                        default=0, type=int,
                        help="Marker channel (zero indexed)")

    args = parser.parse_args()

    generate_projections_from_microscope_image(args.input_file,
                                               args.wall_channel,
                                               args.marker_channel)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import os.path
import sys
import html
import glob
import re

gcov_out = 'gcov.out'
htmlcov_dir = './htmlcov'

CSS = ''''''

TMPL_HEAD = '''<!doctype html>

    {css}
    <title>jcaslib tests coverage - {title}</title>

'''

TMPL_TAIL = '''

'''

TMPL_FILE_INFO = '{idx:5}: {lines_exec:15} {name}'

TMPL_CODE_NORMAL = '{lineno:>4}: {content}'

TMPL_CODE_NOEXEC = '{lineno:>4}: {content}'

TMPL_CODE_EXEC = '{lineno:>4}: {content}'

TMPL_GCOV_INFO = '{content}'

def html_head (out_f, title):
    with open (out_f, 'w') as fh:
        print (TMPL_HEAD.format (title = title, css = CSS), file = fh)
        fh.flush ()
        fh.close ()

def html_tail (out_f):
    with open (out_f, 'a') as fh:
        print (TMPL_TAIL, file = fh)
        fh.flush ()
        fh.close ()

def write_summary (funcs, files):
    dst = os.path.join (htmlcov_dir, 'summary.html')

    def files_info (fh):
        print ("files:", len (files), file = fh)
        idx = 0
        for i in files:
            idx += 1
            i['idx'] = idx
            line = TMPL_FILE_INFO.format(**i)
            print (line, file = fh)

    html_head (dst, 'gcov run summary');

    with open (dst, 'a') as fh:
        files_info (fh)
        fh.flush ()
        fh.close ()

    html_tail (dst);

def parse_summary ():

    funcs_data = list()
    files_data = list()

    def new_cur ():
        return {'name': None, 'lines_exec': None}

    with open (gcov_out, 'r') as fh:
        in_function = False
        in_file = False
        cur = None

        for line in fh.readlines ():

            if cur is None:
                if not in_function and line.startswith ('Function '):
                    in_function = True
                    cur = new_cur ()

                if not in_file and line.startswith ('File '):
                    in_file = True
                    cur = new_cur ()

            if in_function:

                if line.strip () == "":
                    in_function = False
                    funcs_data.append (cur)
                    cur = None

                elif line.startswith ('Function '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if in_file:

                if line.strip () == "":
                    in_file = False
                    files_data.append (cur)
                    cur = None

                elif line.startswith ('File '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if cur is not None:

                if line.startswith ('Lines executed:'):
                    try:
                        cur['lines_exec'] = html.escape (
                                line.strip().split(':')[1].strip())
                    except IndexError:
                        cur['lines_exec'] = '[[IndexError]]'

        fh.close ()

    print ("summary: funcs", len (funcs_data), "- files", len (files_data))
    write_summary (funcs_data, files_data)

def write_html (dst, title, gcov):
    html_head (dst, title)
    with open (dst, 'a') as fh:
        for line in gcov['lines']:
            print (line['tmpl'].format (**line['data']), file = fh)
        fh.flush ()
        fh.close ()
    html_tail (dst)

re_gcov_attr_source = re.compile ('^\s*-:\s*0:Source:(.*)$')
re_gcov_attr_runs = re.compile ('^\s*-:\s*0:Runs:(\d*)$')
re_gcov_normal = re.compile ('^\s*-:\s*(\d*):(.*)$')
re_gcov_noexec = re.compile ('^\s*#####:\s*(\d*):(.*)$')
re_gcov_exec = re.compile ('^\s*\d*:\s*(\d*):(.*)$')
re_gcov_info = re.compile ('^(\w+\s.*)$')

def parse_gcov (src):

    def new_code_line (gcov, tmpl, lineno, content):
        gcov['lines'].append ({
            'tmpl': tmpl,
            'data': {'lineno': lineno, 'content': content},
        })

    def new_line (gcov, tmpl, content):
        gcov['lines'].append ({'tmpl': tmpl, 'data': {'content': content}})

    dst = os.path.join (htmlcov_dir, src)
    dst = dst.replace('.gcov', '.html')
    gcov = dict(lines = list ())

    # XXX: not sure why yet but it needs to start as 1 instead of 0
    gcov_lines = 1

    print ("parse:", src, "->", dst)

    with open (src, 'r') as fh:
        for line in fh.readlines ():

            m = None
            gcov_lines += 1

            m = re_gcov_attr_source.match (line)
            if m:
                gcov['attr.source'] = html.escape (m.group (1))
                continue

            m = re_gcov_attr_runs.match (line)
            if m:
                gcov['attr.runs'] = html.escape (m.group (1))
                continue

            m = re_gcov_normal.match (line)
            if m:
                idx = m.group (1)
                if idx != "0":
                    new_code_line (gcov, TMPL_CODE_NORMAL, idx,
                            html.escape (m.group (2)))
                continue

            m = re_gcov_noexec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_NOEXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_exec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_EXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_info.match (line)
            if m:
                new_line (gcov, TMPL_GCOV_INFO, html.escape (m.group (1)))
                continue

            if m is None:
                print ("parse:", src, "unkown line:", gcov_lines)

        fh.close ()
    write_html (dst, src.replace ('.gcov', ''), gcov)
    return gcov

def scan_files ():
    db = list()

    def gcov_append (src, gcov):
        db.append ({'src': src, 'data': gcov})

    for src in sorted (glob.glob ('*.gcov')):
        gcov_append (src, parse_gcov (src))

    return db

def write_index (gcovdb):
    dst = os.path.join (htmlcov_dir, 'index.html')
    html_head (dst, 'index')
    html_tail (dst)
    print ("index:", dst)

def pre_checks ():
    if not os.path.isfile (gcov_out):
        print (gcov_out, "file not found")
        sys.exit (1)
    if not os.path.isdir (htmlcov_dir):
        print (htmlcov_dir, "dir not found")
        sys.exit (1)

def main ():
    pre_checks ()
    gcovdb = scan_files ()
    parse_summary ()
    write_index (gcovdb)

if __name__ == '__main__':
    sys.exit (main())
<commit_msg>rename SUMM vars<commit_after>#!/usr/bin/env python3

import os
import os.path
import sys
import html
import glob
import re

gcov_out = 'gcov.out'
htmlcov_dir = './htmlcov'

CSS = ''''''

TMPL_HEAD = '''<!doctype html>

    {css}
    <title>jcaslib tests coverage - {title}</title>

'''

TMPL_TAIL = '''

'''

TMPL_FILE_SUMM = '{idx:5}: {lines_exec:15} {name}'

TMPL_CODE_NORMAL = '{lineno:>4}: {content}'

TMPL_CODE_NOEXEC = '{lineno:>4}: {content}'

TMPL_CODE_EXEC = '{lineno:>4}: {content}'

TMPL_GCOV_INFO = '{content}'

def html_head (out_f, title):
    with open (out_f, 'w') as fh:
        print (TMPL_HEAD.format (title = title, css = CSS), file = fh)
        fh.flush ()
        fh.close ()

def html_tail (out_f):
    with open (out_f, 'a') as fh:
        print (TMPL_TAIL, file = fh)
        fh.flush ()
        fh.close ()

def write_summary (funcs, files):
    dst = os.path.join (htmlcov_dir, 'summary.html')

    def files_info (fh):
        print ("files:", len (files), file = fh)
        idx = 0
        for i in files:
            idx += 1
            i['idx'] = idx
            line = TMPL_FILE_SUMM.format(**i)
            print (line, file = fh)

    html_head (dst, 'gcov run summary');

    with open (dst, 'a') as fh:
        files_info (fh)
        fh.flush ()
        fh.close ()

    html_tail (dst);

def parse_summary ():

    funcs_data = list()
    files_data = list()

    def new_cur ():
        return {'name': None, 'lines_exec': None}

    with open (gcov_out, 'r') as fh:
        in_function = False
        in_file = False
        cur = None

        for line in fh.readlines ():

            if cur is None:
                if not in_function and line.startswith ('Function '):
                    in_function = True
                    cur = new_cur ()

                if not in_file and line.startswith ('File '):
                    in_file = True
                    cur = new_cur ()

            if in_function:

                if line.strip () == "":
                    in_function = False
                    funcs_data.append (cur)
                    cur = None

                elif line.startswith ('Function '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if in_file:

                if line.strip () == "":
                    in_file = False
                    files_data.append (cur)
                    cur = None

                elif line.startswith ('File '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if cur is not None:

                if line.startswith ('Lines executed:'):
                    try:
                        cur['lines_exec'] = html.escape (
                                line.strip().split(':')[1].strip())
                    except IndexError:
                        cur['lines_exec'] = '[[IndexError]]'

        fh.close ()

    print ("summary: funcs", len (funcs_data), "- files", len (files_data))
    write_summary (funcs_data, files_data)

def write_html (dst, title, gcov):
    html_head (dst, title)
    with open (dst, 'a') as fh:
        for line in gcov['lines']:
            print (line['tmpl'].format (**line['data']), file = fh)
        fh.flush ()
        fh.close ()
    html_tail (dst)

re_gcov_attr_source = re.compile ('^\s*-:\s*0:Source:(.*)$')
re_gcov_attr_runs = re.compile ('^\s*-:\s*0:Runs:(\d*)$')
re_gcov_normal = re.compile ('^\s*-:\s*(\d*):(.*)$')
re_gcov_noexec = re.compile ('^\s*#####:\s*(\d*):(.*)$')
re_gcov_exec = re.compile ('^\s*\d*:\s*(\d*):(.*)$')
re_gcov_info = re.compile ('^(\w+\s.*)$')

def parse_gcov (src):

    def new_code_line (gcov, tmpl, lineno, content):
        gcov['lines'].append ({
            'tmpl': tmpl,
            'data': {'lineno': lineno, 'content': content},
        })

    def new_line (gcov, tmpl, content):
        gcov['lines'].append ({'tmpl': tmpl, 'data': {'content': content}})

    dst = os.path.join (htmlcov_dir, src)
    dst = dst.replace('.gcov', '.html')
    gcov = dict(lines = list ())

    # XXX: not sure why yet but it needs to start as 1 instead of 0
    gcov_lines = 1

    print ("parse:", src, "->", dst)

    with open (src, 'r') as fh:
        for line in fh.readlines ():

            m = None
            gcov_lines += 1

            m = re_gcov_attr_source.match (line)
            if m:
                gcov['attr.source'] = html.escape (m.group (1))
                continue

            m = re_gcov_attr_runs.match (line)
            if m:
                gcov['attr.runs'] = html.escape (m.group (1))
                continue

            m = re_gcov_normal.match (line)
            if m:
                idx = m.group (1)
                if idx != "0":
                    new_code_line (gcov, TMPL_CODE_NORMAL, idx,
                            html.escape (m.group (2)))
                continue

            m = re_gcov_noexec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_NOEXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_exec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_EXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_info.match (line)
            if m:
                new_line (gcov, TMPL_GCOV_INFO, html.escape (m.group (1)))
                continue

            if m is None:
                print ("parse:", src, "unkown line:", gcov_lines)

        fh.close ()
    write_html (dst, src.replace ('.gcov', ''), gcov)
    return gcov

def scan_files ():
    db = list()

    def gcov_append (src, gcov):
        db.append ({'src': src, 'data': gcov})

    for src in sorted (glob.glob ('*.gcov')):
        gcov_append (src, parse_gcov (src))

    return db

def write_index (gcovdb):
    dst = os.path.join (htmlcov_dir, 'index.html')
    html_head (dst, 'index')

    with open (dst, 'a') as fh:
        print ("scanned files:", len (gcovdb), file = fh)
        fh.flush ()
        fh.close ()

    html_tail (dst)
    print ("index:", dst)

def pre_checks ():
    if not os.path.isfile (gcov_out):
        print (gcov_out, "file not found")
        sys.exit (1)
    if not os.path.isdir (htmlcov_dir):
        print (htmlcov_dir, "dir not found")
        sys.exit (1)

def main ():
    pre_checks ()
    gcovdb = scan_files ()
    parse_summary ()
    write_index (gcovdb)

if __name__ == '__main__':
    sys.exit (main())
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import os.path
import sys
import html
import glob
import re

gcov_out = 'gcov.out'
htmlcov_dir = './htmlcov'

#
# -- HTML templates
#

CSS = ''''''

TMPL_HEAD = '''<!doctype html>

    {css}
    <title>jcaslib tests coverage - {title}</title>

'''

TMPL_TAIL = '''

'''

TMPL_FILE_SUMM = '{idx:5}: {lines_exec:15} {name}'

TMPL_CODE_NORMAL = '{lineno:>4}: {content}'

TMPL_CODE_NOEXEC = '{lineno:>4}: {content}'

TMPL_CODE_EXEC = '{lineno:>4}: {content}'

TMPL_GCOV_INFO = '{content}'

TMPL_GCOV_ATTRIB = '{attr_key}: {attr_val}'

TMPL_LINK = '{content}'

#
# -- html helpers
#

def html_link (href, content):
    return TMPL_LINK.format (href = href + '.html', content = content)

def html_navbar ():
    s = TMPL_LINK.format (href = './index.html', content = 'index')
    return s + "\n"

def html_gcov_attribs (src, gcov):
    s = "[no attribs]" + src
    attr_found = False
    for k in sorted (gcov.keys ()):
        if k.startswith ('attr.'):
            if not attr_found:
                attr_found = True
                s = TMPL_GCOV_ATTRIB.format (attr_key = 'gcov', attr_val = src)
            try:
                kn = k.split ('.')[1]
                kv = gcov.get ('attr.' + kn, None)
                s += "\n"
                s += TMPL_GCOV_ATTRIB.format (attr_key = kn, attr_val = kv)
            except IndexError as e:
                print ("gcov_attribs:", src, "IndexError:", str (e))
    return s + "\n"

#
# -- file i/o (write) actions
#

def write_html_head (out_f, title):
    with open (out_f, 'w') as fh:
        print (TMPL_HEAD.format (title = title, css = CSS), file = fh)
        fh.flush ()
        fh.close ()

def write_html_tail (out_f):
    with open (out_f, 'a') as fh:
        print (TMPL_TAIL, file = fh)
        fh.flush ()
        fh.close ()

def write_summary (funcs, files):
    dst = os.path.join (htmlcov_dir, 'summary.html')

    def files_info (fh):
        print ("files:", len (files), file = fh)
        idx = 0
        for i in files:
            idx += 1
            i['idx'] = idx
            line = TMPL_FILE_SUMM.format(**i)
            print (line, file = fh)

    write_html_head (dst, 'gcov run summary');

    with open (dst, 'a') as fh:
        files_info (fh)
        fh.flush ()
        fh.close ()

    write_html_tail (dst);

def write_gcov_html (src, dst, gcov):
    write_html_head (dst, src.replace ('.gcov', ''))
    with open (dst, 'a') as fh:

        print (html_navbar (), file = fh)
        print (html_gcov_attribs (src, gcov), file = fh)

        for line in gcov['lines']:
            print (line['tmpl'].format (**line['data']), file = fh)

        fh.flush ()
        fh.close ()
    write_html_tail (dst)

def write_index (gcovdb):
    dst = os.path.join (htmlcov_dir, 'index.html')
    write_html_head (dst, 'index')

    with open (dst, 'a') as fh:

        print ("scanned files:", len (gcovdb), file = fh)

        for i in gcovdb:
            print ("       ", html_link (i['src'], i['src']), file = fh)

        fh.flush ()
        fh.close ()

    write_html_tail (dst)
    print ("index:", dst)

#
# -- parse gcov info/report files
#

def parse_summary ():

    funcs_data = list()
    files_data = list()

    def new_cur ():
        return {'name': None, 'lines_exec': None}

    with open (gcov_out, 'r') as fh:
        in_function = False
        in_file = False
        cur = None

        for line in fh.readlines ():

            if cur is None:
                if not in_function and line.startswith ('Function '):
                    in_function = True
                    cur = new_cur ()

                if not in_file and line.startswith ('File '):
                    in_file = True
                    cur = new_cur ()

            if in_function:

                if line.strip () == "":
                    in_function = False
                    funcs_data.append (cur)
                    cur = None

                elif line.startswith ('Function '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if in_file:

                if line.strip () == "":
                    in_file = False
                    files_data.append (cur)
                    cur = None

                elif line.startswith ('File '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if cur is not None:

                if line.startswith ('Lines executed:'):
                    try:
                        cur['lines_exec'] = html.escape (
                                line.strip().split(':')[1].strip())
                    except IndexError:
                        cur['lines_exec'] = '[[IndexError]]'

        fh.close ()

    print ("summary: funcs", len (funcs_data), "- files", len (files_data))
    write_summary (funcs_data, files_data)

# -- parse gcov regexs
re_gcov_attr_source = re.compile ('^\s*-:\s*0:Source:(.*)$')
re_gcov_attr_runs = re.compile ('^\s*-:\s*0:Runs:(\d*)$')
re_gcov_normal = re.compile ('^\s*-:\s*(\d*):(.*)$')
re_gcov_noexec = re.compile ('^\s*#####:\s*(\d*):(.*)$')
re_gcov_exec = re.compile ('^\s*\d*:\s*(\d*):(.*)$')
re_gcov_info = re.compile ('^(\w+\s.*)$')

def parse_gcov (src):

    def new_code_line (gcov, tmpl, lineno, content):
        gcov['lines'].append ({
            'tmpl': tmpl,
            'data': {'lineno': lineno, 'content': content},
        })

    def new_line (gcov, tmpl, content):
        gcov['lines'].append ({'tmpl': tmpl, 'data': {'content': content}})

    dst = os.path.join (htmlcov_dir, src)
    dst = dst.replace('.gcov', '.html')
    gcov = dict(lines = list ())

    # XXX: not sure why yet but it needs to start as 1 instead of 0
    gcov_lines = 1

    print ("parse:", src, "->", dst)

    with open (src, 'r') as fh:
        for line in fh.readlines ():

            m = None
            gcov_lines += 1

            m = re_gcov_attr_source.match (line)
            if m:
                gcov['attr.source'] = html.escape (m.group (1))
                continue

            m = re_gcov_attr_runs.match (line)
            if m:
                gcov['attr.runs'] = html.escape (m.group (1))
                continue

            m = re_gcov_normal.match (line)
            if m:
                idx = m.group (1)
                if idx != "0":
                    new_code_line (gcov, TMPL_CODE_NORMAL, idx,
                            html.escape (m.group (2)))
                continue

            m = re_gcov_noexec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_NOEXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_exec.match (line)
            if m:
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_EXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_info.match (line)
            if m:
                new_line (gcov, TMPL_GCOV_INFO, html.escape (m.group (1)))
                continue

            if m is None:
                print ("parse:", src, "unkown line:", gcov_lines)

        fh.close ()
    write_gcov_html (src, dst, gcov)
    return gcov

#
# -- main
#

def scan_files ():
    db = list()

    def gcov_append (src, gcov):
        db.append ({
            'src': html.escape (src.replace ('.gcov', '')),
            'data': gcov,
        })

    for src in sorted (glob.glob ('*.gcov')):
        gcov_append (src, parse_gcov (src))

    return db

def pre_checks ():
    if not os.path.isfile (gcov_out):
        print (gcov_out, "file not found")
        sys.exit (1)
    if not os.path.isdir (htmlcov_dir):
        print (htmlcov_dir, "dir not found")
        sys.exit (1)

def main ():
    pre_checks ()
    gcovdb = scan_files ()
    parse_summary ()
    write_index (gcovdb)

if __name__ == '__main__':
    sys.exit (main())
<commit_msg>show source file lines executed info<commit_after>#!/usr/bin/env python3

import os
import os.path
import sys
import html
import glob
import re

gcov_out = 'gcov.out'
htmlcov_dir = './htmlcov'

#
# -- HTML templates
#

CSS = ''''''

TMPL_HEAD = '''<!doctype html>

    {css}
    <title>jcaslib tests coverage - {title}</title>

'''

TMPL_TAIL = '''

'''

TMPL_FILE_SUMM = '{idx:5}: {lines_exec:15} {name}'

TMPL_CODE_NORMAL = '{lineno:>4}: {content}'

TMPL_CODE_NOEXEC = '{lineno:>4}: {content}'

TMPL_CODE_EXEC = '{lineno:>4}: {content}'

TMPL_GCOV_INFO = '{content}'

TMPL_GCOV_ATTRIB = '{attr_key}: {attr_val}'

TMPL_LINK = '{content}'

#
# -- html helpers
#

def html_link (href, content):
    return TMPL_LINK.format (href = href + '.html', content = content)

def html_navbar ():
    s = TMPL_LINK.format (href = './index.html', content = 'index')
    return s + "\n"

def html_gcov_attribs (src, gcov):
    s = "[no attribs]" + src
    attr_found = False
    for k in sorted (gcov.keys ()):
        if k.startswith ('attr.'):
            if not attr_found:
                attr_found = True
                s = TMPL_GCOV_ATTRIB.format (attr_key = 'gcov', attr_val = src)
            try:
                kn = '.'.join (k.split ('.')[1:])
                kv = gcov.get ('attr.' + kn, None)
                s += "\n"
                s += TMPL_GCOV_ATTRIB.format (attr_key = kn, attr_val = kv)
            except IndexError as e:
                print ("gcov_attribs:", src, "IndexError:", str (e))
    return s + "\n"

#
# -- file i/o (write) actions
#

def write_html_head (out_f, title):
    with open (out_f, 'w') as fh:
        print (TMPL_HEAD.format (title = title, css = CSS), file = fh)
        fh.flush ()
        fh.close ()

def write_html_tail (out_f):
    with open (out_f, 'a') as fh:
        print (TMPL_TAIL, file = fh)
        fh.flush ()
        fh.close ()

def write_summary (funcs, files):
    dst = os.path.join (htmlcov_dir, 'summary.html')

    def files_info (fh):
        print ("files:", len (files), file = fh)
        idx = 0
        for i in files:
            idx += 1
            i['idx'] = idx
            line = TMPL_FILE_SUMM.format(**i)
            print (line, file = fh)

    write_html_head (dst, 'gcov run summary');

    with open (dst, 'a') as fh:
        files_info (fh)
        fh.flush ()
        fh.close ()

    write_html_tail (dst);

def write_gcov_html (src, dst, gcov):
    write_html_head (dst, src.replace ('.gcov', ''))
    with open (dst, 'a') as fh:

        print (html_navbar (), file = fh)
        print (html_gcov_attribs (src, gcov), file = fh)

        for line in gcov['lines']:
            print (line['tmpl'].format (**line['data']), file = fh)

        fh.flush ()
        fh.close ()
    write_html_tail (dst)

def write_index (gcovdb):
    dst = os.path.join (htmlcov_dir, 'index.html')
    write_html_head (dst, 'index')

    with open (dst, 'a') as fh:

        print ("scanned files:", len (gcovdb), file = fh)

        for i in gcovdb:
            print ("       ", html_link (i['src'], i['src']), file = fh)

        fh.flush ()
        fh.close ()

    write_html_tail (dst)
    print ("index:", dst)

#
# -- parse gcov info/report files
#

def parse_summary ():

    funcs_data = list()
    files_data = list()

    def new_cur ():
        return {'name': None, 'lines_exec': None}

    with open (gcov_out, 'r') as fh:
        in_function = False
        in_file = False
        cur = None

        for line in fh.readlines ():

            if cur is None:
                if not in_function and line.startswith ('Function '):
                    in_function = True
                    cur = new_cur ()

                if not in_file and line.startswith ('File '):
                    in_file = True
                    cur = new_cur ()

            if in_function:

                if line.strip () == "":
                    in_function = False
                    funcs_data.append (cur)
                    cur = None

                elif line.startswith ('Function '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if in_file:

                if line.strip () == "":
                    in_file = False
                    files_data.append (cur)
                    cur = None

                elif line.startswith ('File '):
                    try:
                        cur['name'] = html.escape (
                                line.strip().split(' ')[1].replace("'", ""))
                    except IndexError:
                        cur['name'] = '[[IndexError]]'

            if cur is not None:

                if line.startswith ('Lines executed:'):
                    try:
                        cur['lines_exec'] = html.escape (
                                line.strip().split(':')[1].strip())
                    except IndexError:
                        cur['lines_exec'] = '[[IndexError]]'

        fh.close ()

    print ("summary: funcs", len (funcs_data), "- files", len (files_data))
    write_summary (funcs_data, files_data)

# -- parse gcov regexs
re_gcov_attr_source = re.compile ('^\s*-:\s*0:Source:(.*)$')
re_gcov_attr_runs = re.compile ('^\s*-:\s*0:Runs:(\d*)$')
re_gcov_normal = re.compile ('^\s*-:\s*(\d*):(.*)$')
re_gcov_noexec = re.compile ('^\s*#####:\s*(\d*):(.*)$')
re_gcov_exec = re.compile ('^\s*\d*:\s*(\d*):(.*)$')
re_gcov_info = re.compile ('^(\w+\s.*)$')

def parse_gcov (src):

    def new_code_line (gcov, tmpl, lineno, content):
        gcov['lines'].append ({
            'tmpl': tmpl,
            'data': {'lineno': lineno, 'content': content},
        })

    def new_line (gcov, tmpl, content):
        gcov['lines'].append ({'tmpl': tmpl, 'data': {'content': content}})

    def update_attribs (gcov, attr):
        for k in attr.keys ():
            gcov['attr.' + k] = attr.get (k)

    dst = os.path.join (htmlcov_dir, src)
    dst = dst.replace('.gcov', '.html')
    gcov = dict(lines = list ())

    # XXX: not sure why yet but it needs to start as 1 instead of 0
    gcov_lines = 1
    attr = {
        'source.lines': 0,
        'source.lines.exec': 0,
        'source.lines.noexec': 0,
    }

    print ("parse:", src, "->", dst)

    with open (src, 'r') as fh:
        for line in fh.readlines ():

            m = None
            gcov_lines += 1

            m = re_gcov_attr_source.match (line)
            if m:
                gcov['attr.source'] = html.escape (m.group (1))
                continue

            m = re_gcov_attr_runs.match (line)
            if m:
                gcov['attr.runs'] = html.escape (m.group (1))
                continue

            m = re_gcov_normal.match (line)
            if m:
                idx = m.group (1)
                if idx != "0":
                    attr['source.lines'] += 1
                    new_code_line (gcov, TMPL_CODE_NORMAL, idx,
                            html.escape (m.group (2)))
                continue

            m = re_gcov_noexec.match (line)
            if m:
                attr['source.lines'] += 1
                attr['source.lines.noexec'] += 1
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_NOEXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_exec.match (line)
            if m:
                attr['source.lines'] += 1
                attr['source.lines.exec'] += 1
                idx = m.group (1)
                new_code_line (gcov, TMPL_CODE_EXEC, idx,
                        html.escape (m.group (2)))
                continue

            m = re_gcov_info.match (line)
            if m:
                new_line (gcov, TMPL_GCOV_INFO, html.escape (m.group (1)))
                continue

            if m is None:
                print ("parse:", src, "unkown line:", gcov_lines)

        fh.close ()

    update_attribs (gcov, attr)
    write_gcov_html (src, dst, gcov)

    return gcov

#
# -- main
#

def scan_files ():
    db = list()

    def gcov_append (src, gcov):
        db.append ({
            'src': html.escape (src.replace ('.gcov', '')),
            'data': gcov,
        })

    for src in sorted (glob.glob ('*.gcov')):
        gcov_append (src, parse_gcov (src))

    return db

def pre_checks ():
    if not os.path.isfile (gcov_out):
        print (gcov_out, "file not found")
        sys.exit (1)
    if not os.path.isdir (htmlcov_dir):
        print (htmlcov_dir, "dir not found")
        sys.exit (1)

def main ():
    pre_checks ()
    gcovdb = scan_files ()
    parse_summary ()
    write_index (gcovdb)

if __name__ == '__main__':
    sys.exit (main())
<|endoftext|>


<commit_before>"""
This script creates validation plots based on statistics collected by
validstats.py module.
Statfile structure: first line - any comment (will be ignored), other lines -
values, separated with tabs (date, unrev_main, old_main, unrev_file, old_file,
unrev_template, old_template, unrev_cat, old_cat, unrev_redir, old_redir).

Usage:
    python validplots.py statfile [outdir]

outdir is used for temporary file storing.
"""

import sys
import os
import pywikibot
import matplotlib
matplotlib.use('AGG')
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator

FILEDESC = """
== Краткое описание ==
{{Изображение
| Описание       = Динамика числа неотпатрулированных страниц в виде графика.
| Источник       = собственная работа
| Время создания = 2017
| Автор          = [[У:Facenapalm]]
}}

== Лицензирование ==
{{self|CC-Zero}}
"""

LOCAL = False # set it to True to deny file uploading
DAYS = 365 # -1 to draw plot using all data

def filter_date(date):
    """Return x-axis labels based on dates list."""
    if date[:2] != "01":
        return ""
    return date[3:]

def main():
    """Main script function."""
    argc = len(sys.argv)
    if argc == 1:
        return
    fname = sys.argv[1]
    if argc == 2:
        tempcat = "."
    else:
        tempcat = sys.argv[2]

    raw = open(fname).read()
    data = [line.split("\t") for line in raw.split("\n")[1:-1]]
    data = [list(line) for line in zip(*data)] # transpose data
    if DAYS != -1:
        data = [line[-DAYS:] for line in data]
    data[0] = [filter_date(date) for date in data[0]]
    axis = list(range(len(data[0])))

    if not LOCAL:
        site = pywikibot.Site()

    def _plot_pair(title, ufilename, udata, ucolor, ofilename, odata, ocolor):
        """
        Make 2 plots with unreviewed and old pages respectively.
        If ufilename == ofilename, then make it on single canvas.
        """
        utitle = "Динамика числа неотпатрулированных {}".format(title)
        otitle = "Динамика числа устаревших {}".format(title)
        single = ufilename == ofilename

        def _init_plot(title):
            """Iternal function for plt initialization."""
            plt.figure(figsize=(16, 9), dpi=100)
            plt.xticks(axis, data[0])
            plt.xlabel(title)
            plt.gca().yaxis.set_major_locator(MaxNLocator(integer=True))

        def _line_plot(data, color, label):
            """Iternal function for making one plot line."""
            plt.plot(axis, data, linewidth=3, color=color, label=label)

        def _final_plot(filename):
            """Iternal function for plot saving and uploading."""
            filepath = os.path.join(tempcat, filename)
            if single:
                plt.legend(loc=2)
            plt.margins(0, 0.02)
            plt.subplots_adjust(left=0.05, right=1, top=1, bottom=0.1)
            plt.savefig(filepath, bbox_inches="tight")

            if not LOCAL:
                page = pywikibot.FilePage(site, filename)
                page.upload(filepath,
                            comment="Обновление графика.",
                            text=FILEDESC,
                            ignore_warnings=True)

                os.remove(filepath)

        _init_plot(utitle)
        _line_plot(udata, ucolor, "Неотпатрулированные")
        if not single:
            _final_plot(ufilename)
            _init_plot(otitle)
        _line_plot(odata, ocolor, "Устаревшие")
        _final_plot(ofilename)

    def _backup_data(pagename):
        if not LOCAL:
            page = pywikibot.Page(site, pagename)
            page.text = "
\n{}\n
".format(raw)
            page.save("Бекап статистики.")

    _plot_pair("статей",
               "validation main unrev.png", data[1], "#2A4B8D",
               "validation main old.png", data[2], "#3366CC")
    _plot_pair("файлов",
               "validation files unrev.png", data[3], "#B32424",
               "validation files old.png", data[4], "#DD3333")
    _plot_pair("шаблонов",
               "validation templates unrev.png", data[5], "#72777D",
               "validation templates old.png", data[6], "#A2A9B1")
    _plot_pair("категорий",
               "validation categories.png", data[7], "#AC6600",
               "validation categories.png", data[8], "#FFCC33")
    _plot_pair("перенаправлений",
               "validation redirects unrev.png", data[9], "#14866D",
               "validation redirects old.png", data[10], "#00AF89")
    _backup_data("User:NapalmBot/validation.tsv")

if __name__ == "__main__":
    main()
<commit_msg>validplots.py: date fix<commit_after>"""
This script creates validation plots based on statistics collected by
validstats.py module.
Statfile structure: first line - any comment (will be ignored), other lines -
values, separated with tabs (date, unrev_main, old_main, unrev_file, old_file,
unrev_template, old_template, unrev_cat, old_cat, unrev_redir, old_redir).

Usage:
    python validplots.py statfile [outdir]

outdir is used for temporary file storing.
"""

import sys
import os
import pywikibot
import matplotlib
matplotlib.use('AGG')
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator

FILEDESC = """
== Краткое описание ==
{{Изображение
| Описание       = Динамика числа неотпатрулированных страниц в виде графика.
| Источник       = собственная работа
| Время создания = 2017
| Автор          = [[У:Facenapalm]]
}}

== Лицензирование ==
{{self|CC-Zero}}
"""

LOCAL = False # set it to True to deny file uploading
DAYS = 365 # -1 to draw plot using all data

def filter_date(date):
    """Return x-axis labels based on dates list."""
    if date[-2:] != "01":
        return ""
    return date[:7]

def main():
    """Main script function."""
    argc = len(sys.argv)
    if argc == 1:
        return
    fname = sys.argv[1]
    if argc == 2:
        tempcat = "."
    else:
        tempcat = sys.argv[2]

    raw = open(fname).read()
    data = [line.split("\t") for line in raw.split("\n")[1:-1]]
    data = [list(line) for line in zip(*data)] # transpose data
    if DAYS != -1:
        data = [line[-DAYS:] for line in data]
    data[0] = [filter_date(date) for date in data[0]]
    axis = list(range(len(data[0])))

    if not LOCAL:
        site = pywikibot.Site()

    def _plot_pair(title, ufilename, udata, ucolor, ofilename, odata, ocolor):
        """
        Make 2 plots with unreviewed and old pages respectively.
        If ufilename == ofilename, then make it on single canvas.
        """
        utitle = "Динамика числа неотпатрулированных {}".format(title)
        otitle = "Динамика числа устаревших {}".format(title)
        single = ufilename == ofilename

        def _init_plot(title):
            """Iternal function for plt initialization."""
            plt.figure(figsize=(16, 9), dpi=100)
            plt.xticks(axis, data[0])
            plt.xlabel(title)
            plt.gca().yaxis.set_major_locator(MaxNLocator(integer=True))

        def _line_plot(data, color, label):
            """Iternal function for making one plot line."""
            plt.plot(axis, data, linewidth=3, color=color, label=label)

        def _final_plot(filename):
            """Iternal function for plot saving and uploading."""
            filepath = os.path.join(tempcat, filename)
            if single:
                plt.legend(loc=2)
            plt.margins(0, 0.02)
            plt.subplots_adjust(left=0.05, right=1, top=1, bottom=0.1)
            plt.savefig(filepath, bbox_inches="tight")

            if not LOCAL:
                page = pywikibot.FilePage(site, filename)
                page.upload(filepath,
                            comment="Обновление графика.",
                            text=FILEDESC,
                            ignore_warnings=True)

                os.remove(filepath)

        _init_plot(utitle)
        _line_plot(udata, ucolor, "Неотпатрулированные")
        if not single:
            _final_plot(ufilename)
            _init_plot(otitle)
        _line_plot(odata, ocolor, "Устаревшие")
        _final_plot(ofilename)

    def _backup_data(pagename):
        if not LOCAL:
            page = pywikibot.Page(site, pagename)
            page.text = "
\n{}\n
".format(raw)
            page.save("Бекап статистики.")

    _plot_pair("статей",
               "validation main unrev.png", data[1], "#2A4B8D",
               "validation main old.png", data[2], "#3366CC")
    _plot_pair("файлов",
               "validation files unrev.png", data[3], "#B32424",
               "validation files old.png", data[4], "#DD3333")
    _plot_pair("шаблонов",
               "validation templates unrev.png", data[5], "#72777D",
               "validation templates old.png", data[6], "#A2A9B1")
    _plot_pair("категорий",
               "validation categories.png", data[7], "#AC6600",
               "validation categories.png", data[8], "#FFCC33")
    _plot_pair("перенаправлений",
               "validation redirects unrev.png", data[9], "#14866D",
               "validation redirects old.png", data[10], "#00AF89")
    _backup_data("User:NapalmBot/validation.tsv")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This script can move pages.

These command line parameters can be used to specify which pages to work on:

&params;

Furthermore, the following command line parameters are supported:

-from and -to     The page to move from and the page to move to.

-noredirect       Leave no redirect behind.

-notalkpage       Do not move this page's talk page (if it exists)

-prefix           Move pages by adding a namespace prefix to the names of the
                  pages. (Will remove the old namespace prefix if any)
                  Argument can also be given as "-prefix:namespace:".

-always           Don't prompt to make changes, just do them.

-skipredirects    Skip redirect pages (Warning: increases server load)

-summary          Prompt for a custom summary, bypassing the predefined message
                  texts. Argument can also be given as "-summary:XYZ".

-pairsfile        Read pairs of file names from a file. The file must be in a
                  format [[frompage]] [[topage]] [[frompage]] [[topage]] ...
                  Argument can also be given as "-pairsfile:filename"

"""
#
# (C) Pywikibot team, 2006-2020
#
# Distributed under the terms of the MIT license.
#
import re

import pywikibot

from pywikibot import i18n, pagegenerators
from pywikibot.bot import MultipleSitesBot

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {'&params;': pagegenerators.parameterHelp}  # noqa: N816

class MovePagesBot(MultipleSitesBot):

    """Page move bot."""

    def __init__(self, generator, **kwargs) -> None:
        """Initializer."""
        self.availableOptions.update({
            'prefix': None,
            'noredirect': False,
            'movetalkpage': True,
            'skipredirects': False,
            'summary': None,
        })
        super().__init__(**kwargs)

        self.generator = generator
        self.appendAll = False
        self.regexAll = False
        self.noNamespace = False

    def moveOne(self, page, newPageTitle) -> None:
        """Move on page to newPageTitle."""
        try:
            msg = self.getOption('summary')
            if not msg:
                msg = i18n.twtranslate(page.site, 'movepages-moving')
            pywikibot.output('Moving page {0} to [[{1}]]'
                             .format(page.title(as_link=True), newPageTitle))
            page.move(
                newPageTitle, reason=msg,
                movetalk=self.getOption('movetalkpage'),
                noredirect=self.getOption('noredirect'))
        except pywikibot.PageRelatedError as error:
            pywikibot.output(error)

    def treat(self, page) -> None:
        """Treat a single page."""
        self.current_page = page
        if self.getOption('skipredirects') and page.isRedirectPage():
            pywikibot.output('Page {0} is a redirect; skipping.'
                             .format(page.title()))
            return
        pagetitle = page.title(with_ns=False)
        namesp = page.site.namespace(page.namespace())
        if self.appendAll:
            newPageTitle = ('{0}{1}{2}'.format(self.pagestart, pagetitle,
                                               self.pageend))
            if not self.noNamespace and namesp:
                newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
        elif self.regexAll:
            newPageTitle = self.regex.sub(self.replacePattern, pagetitle)
            if not self.noNamespace and namesp:
                newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
        if self.getOption('prefix'):
            newPageTitle = ('{0}{1}'.format(self.getOption('prefix'),
                                            pagetitle))
        if self.getOption('prefix') or self.appendAll or self.regexAll:
            if self.user_confirm('Change the page title to "{0}"?'
                                 .format(newPageTitle)):
                self.moveOne(page, newPageTitle)
        else:
            choice = pywikibot.input_choice('What do you want to do?',
                                            [('change page name', 'c'),
                                             ('append to page name', 'a'),
                                             ('use a regular expression', 'r'),
                                             ('next page', 'n')])
            if choice == 'c':
                newPageTitle = pywikibot.input('New page name:')
                self.moveOne(page, newPageTitle)
            elif choice == 'a':
                self.pagestart = pywikibot.input('Append this to the start:')
                self.pageend = pywikibot.input('Append this to the end:')
                newPageTitle = ('{0}{1}{2}'.format(self.pagestart, pagetitle,
                                                   self.pageend))
                if namesp:
                    if pywikibot.input_yn('Do you want to remove the '
                                          'namespace prefix "{0}:"?'
                                          .format(namesp),
                                          automatic_quit=False):
                        self.noNamespace = True
                    else:
                        newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
                choice2 = pywikibot.input_choice(
                    'Change the page title to "{0}"?'
                    .format(newPageTitle),
                    [('yes', 'y'), ('no', 'n'), ('all', 'a')])
                if choice2 == 'y':
                    self.moveOne(page, newPageTitle)
                elif choice2 == 'a':
                    self.appendAll = True
                    self.moveOne(page, newPageTitle)
            elif choice == 'r':
                searchPattern = pywikibot.input('Enter the search pattern:')
                self.replacePattern = pywikibot.input(
                    'Enter the replace pattern:')
                self.regex = re.compile(searchPattern)
                if page.title() == page.title(with_ns=False):
                    newPageTitle = self.regex.sub(self.replacePattern,
                                                  page.title())
                else:
                    if pywikibot.input_yn('Do you want to remove the '
                                          'namespace prefix "{0}:"?'
                                          .format(namesp),
                                          automatic_quit=False):
                        newPageTitle = self.regex.sub(
                            self.replacePattern, page.title(with_ns=False))
                        self.noNamespace = True
                    else:
                        newPageTitle = self.regex.sub(self.replacePattern,
                                                      page.title())
                choice2 = pywikibot.input_choice(
                    'Change the page title to "{0}"?'
                    .format(newPageTitle),
                    [('yes', 'y'), ('no', 'n'), ('all', 'a')])
                if choice2 == 'y':
                    self.moveOne(page, newPageTitle)
                elif choice2 == 'a':
                    self.regexAll = True
                    self.moveOne(page, newPageTitle)

def main(*args) -> None:
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: str
    """
    oldName = None
    options = {}
    fromToPairs = []

    # Process global args and prepare generator args parser
    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()

    for arg in local_args:
        if genFactory.handleArg(arg):
            continue
        if arg.startswith('-pairsfile'):
            if len(arg) == len('-pairsfile'):
                filename = pywikibot.input(
                    'Enter the name of the file containing pairs:')
            else:
                filename = arg[len('-pairsfile:'):]
            oldName1 = None
            for page in pagegenerators.TextfilePageGenerator(filename):
                if oldName1:
                    fromToPairs.append([oldName1, page.title()])
                    oldName1 = None
                else:
                    oldName1 = page.title()
            if oldName1:
                pywikibot.warning(
                    'file {0} contains odd number of links'.format(filename))
        elif arg == '-noredirect':
            options['noredirect'] = True
        elif arg == '-notalkpage':
            options['movetalkpage'] = False
        elif arg == '-always':
            options['always'] = True
        elif arg == '-skipredirects':
            options['skipredirects'] = True
        elif arg.startswith('-from:'):
            if oldName:
                pywikibot.warning('-from:{0} without -to:'.format(oldName))
            oldName = arg[len('-from:'):]
        elif arg.startswith('-to:'):
            if oldName:
                fromToPairs.append([oldName, arg[len('-to:'):]])
                oldName = None
            else:
                pywikibot.warning('{0} without -from'.format(arg))
        elif arg.startswith('-prefix'):
            if len(arg) == len('-prefix'):
                options['prefix'] = pywikibot.input('Enter the prefix:')
            else:
                options['prefix'] = arg[8:]
        elif arg.startswith('-summary'):
            if len(arg) == len('-summary'):
                options['summary'] = pywikibot.input('Enter the summary:')
            else:
                options['summary'] = arg[9:]

    if oldName:
        pywikibot.warning('-from:{0} without -to:'.format(oldName))
    site = pywikibot.Site()
    for pair in fromToPairs:
        page = pywikibot.Page(site, pair[0])
        bot = MovePagesBot(None, **options)
        bot.moveOne(page, pair[1])

    gen = genFactory.getCombinedGenerator(preload=True)
    if gen:
        bot = MovePagesBot(gen, **options)
        bot.run()
    elif not fromToPairs:
        pywikibot.bot.suggest_help(missing_generator=True)

if __name__ == '__main__':
    main()
<commit_msg>[IMPR] switch to new OptionHandler interface<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This script can move pages.

These command line parameters can be used to specify which pages to work on:

&params;

Furthermore, the following command line parameters are supported:

-from and -to     The page to move from and the page to move to.

-noredirect       Leave no redirect behind.

-notalkpage       Do not move this page's talk page (if it exists)

-prefix           Move pages by adding a namespace prefix to the names of the
                  pages. (Will remove the old namespace prefix if any)
                  Argument can also be given as "-prefix:namespace:".

-always           Don't prompt to make changes, just do them.

-skipredirects    Skip redirect pages (Warning: increases server load)

-summary          Prompt for a custom summary, bypassing the predefined message
                  texts. Argument can also be given as "-summary:XYZ".

-pairsfile        Read pairs of file names from a file. The file must be in a
                  format [[frompage]] [[topage]] [[frompage]] [[topage]] ...
                  Argument can also be given as "-pairsfile:filename"

"""
#
# (C) Pywikibot team, 2006-2020
#
# Distributed under the terms of the MIT license.
#
import re

import pywikibot

from pywikibot import i18n, pagegenerators
from pywikibot.bot import MultipleSitesBot

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {'&params;': pagegenerators.parameterHelp}  # noqa: N816

class MovePagesBot(MultipleSitesBot):

    """Page move bot."""

    def __init__(self, generator, **kwargs) -> None:
        """Initializer."""
        self.available_options.update({
            'prefix': None,
            'noredirect': False,
            'movetalkpage': True,
            'skipredirects': False,
            'summary': None,
        })
        super().__init__(**kwargs)

        self.generator = generator
        self.appendAll = False
        self.regexAll = False
        self.noNamespace = False

    def moveOne(self, page, newPageTitle) -> None:
        """Move on page to newPageTitle."""
        try:
            msg = self.opt.summary
            if not msg:
                msg = i18n.twtranslate(page.site, 'movepages-moving')
            pywikibot.output('Moving page {0} to [[{1}]]'
                             .format(page.title(as_link=True), newPageTitle))
            page.move(
                newPageTitle, reason=msg,
                movetalk=self.opt.movetalkpage,
                noredirect=self.opt.noredirect)
        except pywikibot.PageRelatedError as error:
            pywikibot.output(error)

    def treat(self, page) -> None:
        """Treat a single page."""
        self.current_page = page
        if self.opt.skipredirects and page.isRedirectPage():
            pywikibot.output('Page {0} is a redirect; skipping.'
                             .format(page.title()))
            return
        pagetitle = page.title(with_ns=False)
        namesp = page.site.namespace(page.namespace())
        if self.appendAll:
            newPageTitle = ('{0}{1}{2}'.format(self.pagestart, pagetitle,
                                               self.pageend))
            if not self.noNamespace and namesp:
                newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
        elif self.regexAll:
            newPageTitle = self.regex.sub(self.replacePattern, pagetitle)
            if not self.noNamespace and namesp:
                newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
        if self.opt.prefix:
            newPageTitle = ('{0}{1}'.format(self.opt.prefix,
                                            pagetitle))
        if self.opt.prefix or self.appendAll or self.regexAll:
            if self.user_confirm('Change the page title to "{0}"?'
                                 .format(newPageTitle)):
                self.moveOne(page, newPageTitle)
        else:
            choice = pywikibot.input_choice('What do you want to do?',
                                            [('change page name', 'c'),
                                             ('append to page name', 'a'),
                                             ('use a regular expression', 'r'),
                                             ('next page', 'n')])
            if choice == 'c':
                newPageTitle = pywikibot.input('New page name:')
                self.moveOne(page, newPageTitle)
            elif choice == 'a':
                self.pagestart = pywikibot.input('Append this to the start:')
                self.pageend = pywikibot.input('Append this to the end:')
                newPageTitle = ('{0}{1}{2}'.format(self.pagestart, pagetitle,
                                                   self.pageend))
                if namesp:
                    if pywikibot.input_yn('Do you want to remove the '
                                          'namespace prefix "{0}:"?'
                                          .format(namesp),
                                          automatic_quit=False):
                        self.noNamespace = True
                    else:
                        newPageTitle = ('{0}:{1}'.format(namesp, newPageTitle))
                choice2 = pywikibot.input_choice(
                    'Change the page title to "{0}"?'
                    .format(newPageTitle),
                    [('yes', 'y'), ('no', 'n'), ('all', 'a')])
                if choice2 == 'y':
                    self.moveOne(page, newPageTitle)
                elif choice2 == 'a':
                    self.appendAll = True
                    self.moveOne(page, newPageTitle)
            elif choice == 'r':
                searchPattern = pywikibot.input('Enter the search pattern:')
                self.replacePattern = pywikibot.input(
                    'Enter the replace pattern:')
                self.regex = re.compile(searchPattern)
                if page.title() == page.title(with_ns=False):
                    newPageTitle = self.regex.sub(self.replacePattern,
                                                  page.title())
                else:
                    if pywikibot.input_yn('Do you want to remove the '
                                          'namespace prefix "{0}:"?'
                                          .format(namesp),
                                          automatic_quit=False):
                        newPageTitle = self.regex.sub(
                            self.replacePattern, page.title(with_ns=False))
                        self.noNamespace = True
                    else:
                        newPageTitle = self.regex.sub(self.replacePattern,
                                                      page.title())
                choice2 = pywikibot.input_choice(
                    'Change the page title to "{0}"?'
                    .format(newPageTitle),
                    [('yes', 'y'), ('no', 'n'), ('all', 'a')])
                if choice2 == 'y':
                    self.moveOne(page, newPageTitle)
                elif choice2 == 'a':
                    self.regexAll = True
                    self.moveOne(page, newPageTitle)

def main(*args) -> None:
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: str
    """
    oldName = None
    options = {}
    fromToPairs = []

    # Process global args and prepare generator args parser
    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()

    for arg in local_args:
        if genFactory.handleArg(arg):
            continue
        if arg.startswith('-pairsfile'):
            if len(arg) == len('-pairsfile'):
                filename = pywikibot.input(
                    'Enter the name of the file containing pairs:')
            else:
                filename = arg[len('-pairsfile:'):]
            oldName1 = None
            for page in pagegenerators.TextfilePageGenerator(filename):
                if oldName1:
                    fromToPairs.append([oldName1, page.title()])
                    oldName1 = None
                else:
                    oldName1 = page.title()
            if oldName1:
                pywikibot.warning(
                    'file {0} contains odd number of links'.format(filename))
        elif arg == '-noredirect':
            options['noredirect'] = True
        elif arg == '-notalkpage':
            options['movetalkpage'] = False
        elif arg == '-always':
            options['always'] = True
        elif arg == '-skipredirects':
            options['skipredirects'] = True
        elif arg.startswith('-from:'):
            if oldName:
                pywikibot.warning('-from:{0} without -to:'.format(oldName))
            oldName = arg[len('-from:'):]
        elif arg.startswith('-to:'):
            if oldName:
                fromToPairs.append([oldName, arg[len('-to:'):]])
                oldName = None
            else:
                pywikibot.warning('{0} without -from'.format(arg))
        elif arg.startswith('-prefix'):
            if len(arg) == len('-prefix'):
                options['prefix'] = pywikibot.input('Enter the prefix:')
            else:
                options['prefix'] = arg[8:]
        elif arg.startswith('-summary'):
            if len(arg) == len('-summary'):
                options['summary'] = pywikibot.input('Enter the summary:')
            else:
                options['summary'] = arg[9:]

    if oldName:
        pywikibot.warning('-from:{0} without -to:'.format(oldName))
    site = pywikibot.Site()
    for pair in fromToPairs:
        page = pywikibot.Page(site, pair[0])
        bot = MovePagesBot(None, **options)
        bot.moveOne(page, pair[1])

    gen = genFactory.getCombinedGenerator(preload=True)
    if gen:
        bot = MovePagesBot(gen, **options)
        bot.run()
    elif not fromToPairs:
        pywikibot.bot.suggest_help(missing_generator=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#  Allows ConnectBox to send status and receive updates from Chathost APIs

print ("phonehome: Starting...")

import requests
import json
import subprocess
import uuid

# Retrieve brand.txt
f = open('/usr/local/connectbox/brand.txt',)
brand = json.load(f)
print (brand)

# Sanity Checks
error = 0
if len(brand['server_url']) < 5:
  print ("FATAL: No server_url")
  error=1
if len(brand['server_sitename']) < 1:
  print ("FATAL: No server_sitename")
  error=1
if error == 1:
  exit(1)

# Get boxId
results = subprocess.run(["cat", "/sys/class/net/eth0/address"], stdout=subprocess.PIPE)
boxId = results.stdout.decode('utf-8')
boxId = boxId.replace(":","-")
boxId = boxId.replace("\n","")
print ("boxId: " + boxId)

# Get authorization token
if len(brand["server_authorization"]) < 8:
  brand["server_authorization"] = str(uuid.uuid4())
  print ("No server_authorization so we generated a GUID: " + brand["server_authorization"])
  # Save token to json
  with open('/usr/local/connectbox/brand.txt', 'w', encoding='utf-8') as f:
    json.dump(brand, f, ensure_ascii=False, indent=4)
  print ("Saved authorization to brand.txt")
token = "Bearer " + boxId + "|" + brand["server_authorization"]
print ("token: " + token)

headers = {"Authorization": token}

def processSettings(settings):
  print ("processSettings: Start")
  print (settings)
  for setting in settings:
    command = "set " + setting["key"] + " " + setting["value"];
    if setting["key"] == 'authorization':
      command = "set brand server_" + setting["key"] + " " + setting["value"];
    print ("Handling Setting: " + command)
    results = subprocess.check_output("sudo /usr/local/connectbox/bin/ConnectBoxManage.sh " + command, shell=True)
    print(results.decode('utf-8'))
    if results.decode('utf-8') == "SUCCESS\n":
      print ("Setting For " + command + " was successful! Now inform server to delete setting")
      response = requests.delete(brand["server_url"] + "/chathost/settings/" + setting["deleteId"],headers=headers)
      if response.status_code == 200:
        print ("phonehome: Successful delete of setting")
      else:
        print ("FATAL: Can't Delete Setting " + brand["server_url"])
        exit(1)

# Main Operation Here

# Initial Trail Connection To Server
response = requests.get(brand["server_url"] + "/chathost/healthcheck", headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful connection to healthcheck")
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

# TODO: Send logs

# Send roster object
data = []
record = {
	"id":1,
	"course_name":brand["Brand"],
	"students":[],
	"teachers":[],
	"notMoodle": True,
	"sitename":brand["server_sitename"],
	"siteadmin_name":brand["server_siteadmin_name"],
	"siteadmin_email":brand["server_siteadmin_email"],
	"siteadmin_phone":brand["server_siteadmin_phone"],
}
data.append(record)
print (data)
response = requests.post(brand["server_url"] + "/chathost/courseRosters", json = data, headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful post to /courseRosters")
elif response.status_code == 401:
	print ("FATAL: Unauthorized to " + brand["server_url"])
	exit(1)
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

# Get Settings
response = requests.get(brand["server_url"] + "/chathost/settings", headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful connection to settings:")
	settings = response.json()
	if len(settings) > 0:
	  print ("Handling " + str(len(settings)) + " settings")
	  processSettings(settings)
	else:
	  print ("No settings to handle")
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

print ("phonehome: Done")

<commit_msg>Fix errors on handling settings<commit_after>#  Allows ConnectBox to send status and receive updates from Chathost APIs

print ("phonehome: Starting...")

import requests
import json
import subprocess
import uuid

# Retrieve brand.txt
f = open('/usr/local/connectbox/brand.txt',)
brand = json.load(f)
print (brand)

# Sanity Checks
error = 0
if len(brand['server_url']) < 5:
  print ("FATAL: No server_url")
  error=1
if len(brand['server_sitename']) < 1:
  print ("FATAL: No server_sitename")
  error=1
if error == 1:
  exit(1)

# Get boxId
results = subprocess.run(["cat", "/sys/class/net/eth0/address"], stdout=subprocess.PIPE)
boxId = results.stdout.decode('utf-8')
boxId = boxId.replace(":","-")
boxId = boxId.replace("\n","")
print ("boxId: " + boxId)

# Get authorization token
if len(brand["server_authorization"]) < 8:
  brand["server_authorization"] = str(uuid.uuid4())
  print ("No server_authorization so we generated a GUID: " + brand["server_authorization"])
  # Save token to json
  with open('/usr/local/connectbox/brand.txt', 'w', encoding='utf-8') as f:
    json.dump(brand, f, ensure_ascii=False, indent=4)
  print ("Saved authorization to brand.txt")
token = "Bearer " + boxId + "|" + brand["server_authorization"]
print ("token: " + token)

headers = {"Authorization": token}

def processSettings(settings):
  print ("processSettings: Start")
  print (settings)
  for setting in settings:
    command = "set " + setting["key"] + " " + setting["value"];
    if setting["key"] == 'authorization':
      command = "set brand server_" + setting["key"] + " " + setting["value"];
    print ("Handling Setting: " + command)
	try:
      results = subprocess.check_output("sudo /usr/local/connectbox/bin/ConnectBoxManage.sh " + command, shell=True)
      results = results.decode('utf-8')
    except:
      print ("Could not process the setting: " + command)
      results = "Null (ConnectBoxManage.sh returned fatal)"
    print("ConnectBoxManage.sh returned: " + results)
    if results == "SUCCESS\n":
      print ("Setting For " + command + " was successful! Now inform server to delete setting")
      response = requests.delete(brand["server_url"] + "/chathost/settings/" + setting["deleteId"],headers=headers)
      if response.status_code == 200:
        print ("phonehome: Successful delete of setting")
      else:
        print ("FATAL: Can't Delete Setting " + brand["server_url"])
        exit(1)

# Main Operation Here

# Initial Trail Connection To Server
response = requests.get(brand["server_url"] + "/chathost/healthcheck", headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful connection to healthcheck")
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

# TODO: Send logs

# Send roster object
data = []
record = {
	"id":1,
	"course_name":brand["Brand"],
	"students":[],
	"teachers":[],
	"notMoodle": True,
	"sitename":brand["server_sitename"],
	"siteadmin_name":brand["server_siteadmin_name"],
	"siteadmin_email":brand["server_siteadmin_email"],
	"siteadmin_phone":brand["server_siteadmin_phone"],
}
data.append(record)
print (data)
response = requests.post(brand["server_url"] + "/chathost/courseRosters", json = data, headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful post to /courseRosters")
elif response.status_code == 401:
	print ("FATAL: Unauthorized to " + brand["server_url"])
	exit(1)
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

# Get Settings
response = requests.get(brand["server_url"] + "/chathost/settings", headers=headers)
if response.status_code == 200:
	print ("phonehome: Successful connection to settings:")
	settings = response.json()
	if len(settings) > 0:
	  print ("Handling " + str(len(settings)) + " settings")
	  processSettings(settings)
	else:
	  print ("No settings to handle")
else:
	print ("FATAL: Can't Connect to " + brand["server_url"])
	exit(1)

print ("phonehome: Done")

<|endoftext|>


<commit_before># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import os
import re
import zipfile

try:
    from StringIO import StringIO
except ImportError:
    from _io import StringIO

from pkginfo import distribution
from pkginfo.distribution import Distribution

# Monkeypatch Metadata 2.0 support
distribution.HEADER_ATTRS_2_0 = distribution.HEADER_ATTRS_1_2
distribution.HEADER_ATTRS.update({"2.0": distribution.HEADER_ATTRS_2_0})

wheel_file_re = re.compile(
    r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
        ((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
        \.whl|\.dist-info)$""",
    re.VERBOSE)

class Wheel(Distribution):

    def __init__(self, filename, metadata_version=None):
        self.filename = filename
        self.metadata_version = metadata_version
        self.extractMetadata()

    @property
    def py_version(self):
        wheel_info = wheel_file_re.match(self.filename)
        return wheel_info.group("pyver")

    def read(self):
        fqn = os.path.abspath(os.path.normpath(self.filename))
        if not os.path.exists(fqn):
            raise ValueError('No such file: %s' % fqn)

        if fqn.endswith('.whl'):
            archive = zipfile.ZipFile(fqn)
            names = archive.namelist()

            def read_file(name):
                return archive.read(name)
        else:
            raise ValueError('Not a known archive format: %s' % fqn)

        try:
            tuples = [x.split('/') for x in names if 'METADATA' in x]
            schwarz = sorted([(len(x), x) for x in tuples])
            for path in [x[1] for x in schwarz]:
                candidate = '/'.join(path)
                data = read_file(candidate)
                if b'Metadata-Version' in data:
                    return data
        finally:
            archive.close()

        raise ValueError('No METADATA in archive: %s' % fqn)

    def parse(self, data):
        super(Wheel, self).parse(data)

        fp = StringIO(distribution.must_decode(data))
        msg = distribution.parse(fp)
        self.description = msg.get_payload()
<commit_msg>Use the basename of the wheel filename<commit_after># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import os
import re
import zipfile

try:
    from StringIO import StringIO
except ImportError:
    from _io import StringIO

from pkginfo import distribution
from pkginfo.distribution import Distribution

# Monkeypatch Metadata 2.0 support
distribution.HEADER_ATTRS_2_0 = distribution.HEADER_ATTRS_1_2
distribution.HEADER_ATTRS.update({"2.0": distribution.HEADER_ATTRS_2_0})

wheel_file_re = re.compile(
    r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
        ((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
        \.whl|\.dist-info)$""",
    re.VERBOSE)

class Wheel(Distribution):

    def __init__(self, filename, metadata_version=None):
        self.filename = filename
        self.basefilename = os.path.basename(self.filename)
        self.metadata_version = metadata_version
        self.extractMetadata()

    @property
    def py_version(self):
        wheel_info = wheel_file_re.match(self.basefilename)
        return wheel_info.group("pyver")

    def read(self):
        fqn = os.path.abspath(os.path.normpath(self.filename))
        if not os.path.exists(fqn):
            raise ValueError('No such file: %s' % fqn)

        if fqn.endswith('.whl'):
            archive = zipfile.ZipFile(fqn)
            names = archive.namelist()

            def read_file(name):
                return archive.read(name)
        else:
            raise ValueError('Not a known archive format: %s' % fqn)

        try:
            tuples = [x.split('/') for x in names if 'METADATA' in x]
            schwarz = sorted([(len(x), x) for x in tuples])
            for path in [x[1] for x in schwarz]:
                candidate = '/'.join(path)
                data = read_file(candidate)
                if b'Metadata-Version' in data:
                    return data
        finally:
            archive.close()

        raise ValueError('No METADATA in archive: %s' % fqn)

    def parse(self, data):
        super(Wheel, self).parse(data)

        fp = StringIO(distribution.must_decode(data))
        msg = distribution.parse(fp)
        self.description = msg.get_payload()
<|endoftext|>