<commit_before>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import hashlib

import os, sys
import zipfile
import glob
import pprint

class KeeperException(Exception):
  pass

class ResourceFilesKeeper():
  """
  This class encapsulates all utility methods for resource files maintenance.
  """

  HOOKS_DIR="hooks"
  PACKAGE_DIR="package"
  STACKS_DIR="stacks"
  COMMON_SERVICES_DIR="common-services"
  CUSTOM_ACTIONS_DIR="custom_actions"
  HOST_SCRIPTS_DIR="host_scripts"
  DASHBOARDS_DIR="dashboards"
  EXTENSIONS_DIR="extensions"

  # For these directories archives are created
  ARCHIVABLE_DIRS = [HOOKS_DIR, PACKAGE_DIR]

  HASH_SUM_FILE=".hash"
  ARCHIVE_NAME="archive.zip"

  PYC_EXT=".pyc"
  METAINFO_XML = "metainfo.xml"

  BUFFER = 1024 * 32

  # Change that to True to see debug output at stderr
  DEBUG=False

  def __init__(self, resources_dir, stacks_dir, verbose=False, nozip=False):
    """
      nozip = create only hash files and skip creating zip archives
    """
    self.resources_dir = resources_dir
    self.stacks_root = stacks_dir
    self.verbose = verbose
    self.nozip = nozip

  def perform_housekeeping(self):
    """
    Performs housekeeping operations on resource files
    """
    self.update_directory_archives()
    # probably, later we will need some additional operations

  def _iter_update_directory_archive(self, subdirs_list):
    for subdir in subdirs_list:
      for root, dirs, _ in os.walk(subdir):
        for d in dirs:
          if d in self.ARCHIVABLE_DIRS:
            full_path = os.path.abspath(os.path.join(root, d))
            self.update_directory_archive(full_path)

  def _update_resources_subdir_archive(self, subdir):
    archive_root = os.path.join(self.resources_dir, subdir)
    self.dbg_out("Updating archive for {0} dir at {1}...".format(subdir, archive_root))

    # update the directories so that the .hash is generated
    self.update_directory_archive(archive_root)

  def update_directory_archives(self):
    """
    Please see AMBARI-4481 for more details
    """
    # archive stacks
    self.dbg_out("Updating archives for stack dirs at {0}...".format(self.stacks_root))
    valid_stacks = self.list_stacks(self.stacks_root)
    self.dbg_out("Stacks: {0}".format(pprint.pformat(valid_stacks)))
    # Iterate over stack directories
    self._iter_update_directory_archive(valid_stacks)

    # archive common services
    common_services_root = os.path.join(self.resources_dir, self.COMMON_SERVICES_DIR)
    self.dbg_out("Updating archives for common services dirs at {0}...".format(common_services_root))
    valid_common_services = self.list_common_services(common_services_root)
    self.dbg_out("Common Services: {0}".format(pprint.pformat(valid_common_services)))
    # Iterate over common services directories
    self._iter_update_directory_archive(valid_common_services)

    # archive extensions
    extensions_root = os.path.join(self.resources_dir, self.EXTENSIONS_DIR)
    self.dbg_out("Updating archives for extensions dirs at {0}...".format(extensions_root))
    valid_extensions = self.list_extensions(extensions_root)
    self.dbg_out("Extensions: {0}".format(pprint.pformat(valid_extensions)))
    # Iterate over extension directories
    self._iter_update_directory_archive(valid_extensions)

    # custom actions
    self._update_resources_subdir_archive(self.CUSTOM_ACTIONS_DIR)

    # agent host scripts
    self._update_resources_subdir_archive(self.HOST_SCRIPTS_DIR)

    # custom service dashboards
    self._update_resources_subdir_archive(self.DASHBOARDS_DIR)

  def _list_metainfo_dirs(self, root_dir):
    valid_items = []  # Format: <stack_dir, ignore(True|False)>
    glob_pattern = "{0}/*/*".format(root_dir)
    dirs = glob.glob(glob_pattern)
    for directory in dirs:
      metainfo_file = os.path.join(directory, self.METAINFO_XML)
      if os.path.exists(metainfo_file):
        valid_items.append(directory)
    return valid_items

  def list_stacks(self, root_dir):
    """
    Builds a list of stack directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list stacks: {0}".format(str(err)))

  def list_common_services(self, root_dir):
    """
    Builds a list of common services directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list common services: {0}".format(str(err)))

  def list_extensions(self, root_dir):
    """
    Builds a list of extension directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list extensions: {0}".format(str(err)))

  def update_directory_archive(self, directory):
    """
    If hash sum for directory is not present or differs from saved value,
    recalculates hash sum and creates directory archive
    """
    skip_empty_directory = True
    cur_hash = self.count_hash_sum(directory)
    saved_hash = self.read_hash_sum(directory)
    if cur_hash != saved_hash:
      if not self.nozip:
        self.zip_directory(directory, skip_empty_directory)
      # Skip generation of .hash file is directory is empty
      if (skip_empty_directory and (not os.path.exists(directory) or not os.listdir(directory))):
        self.dbg_out("Empty directory. Skipping generation of hash file for {0}".format(directory))
      else:
        self.write_hash_sum(directory, cur_hash)
      pass

  def count_hash_sum(self, directory):
    """
    Recursively counts hash sum of all files in directory and subdirectories.
    Files and directories are processed in alphabetical order.
    Ignores previously created directory archives and files containing
    previously calculated hashes. Compiled pyc files are also ignored
    """
    try:
      sha1 = hashlib.sha1()
      file_list = []
      for root, dirs, files in os.walk(directory):
        for f in files:
          if not self.is_ignored(f):
            full_path = os.path.abspath(os.path.join(root, f))
            file_list.append(full_path)
      file_list.sort()
      for path in file_list:
        self.dbg_out("Counting hash of {0}".format(path))
        with open(path, 'rb') as fh:
          while True:
            data = fh.read(self.BUFFER)
            if not data:
              break
            sha1.update(data)
      return sha1.hexdigest()
    except Exception, err:
      raise KeeperException("Can not calculate directory "
                            "hash: {0}".format(str(err)))

  def read_hash_sum(self, directory):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    if os.path.isfile(hash_file):
      try:
        with open(hash_file) as fh:
          return fh.readline().strip()
      except Exception, err:
        raise KeeperException("Can not read file {0} : {1}".format(hash_file,
                                                                   str(err)))
    else:
      return None

  def write_hash_sum(self, directory, new_hash):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file, "w") as fh:
        fh.write(new_hash)
      os.chmod(hash_file, 0o755)
    except Exception, err:
      raise KeeperException("Can not write to file {0} : {1}".format(hash_file,
                                                                   str(err)))

  def zip_directory(self, directory, skip_if_empty = False):
    """
    Packs entire directory into zip file. Hash file is also packaged
    into archive
    """
    self.dbg_out("creating archive for directory {0}".format(directory))
    try:
      if skip_if_empty:
        if not os.path.exists(directory) or not os.listdir(directory):
          self.dbg_out("Empty directory. Skipping archive creation for {0}".format(directory))
          return

      zip_file_path = os.path.join(directory, self.ARCHIVE_NAME)
      zf = zipfile.ZipFile(zip_file_path, "w")
      abs_src = os.path.abspath(directory)
      for root, dirs, files in os.walk(directory):
        for filename in files:
          # Avoid zipping previous archive and hash file and binary pyc files
          if not self.is_ignored(filename):
            absname = os.path.abspath(os.path.join(root, filename))
            arcname = absname[len(abs_src) + 1:]
            self.dbg_out('zipping %s as %s' % (os.path.join(root, filename),
                                        arcname))
            zf.write(absname, arcname)
      zf.close()
      os.chmod(zip_file_path, 0o755)
    except Exception, err:
      raise KeeperException("Can not create zip archive of "
                            "directory {0} : {1}".format(directory, str(err)))

  def is_ignored(self, filename):
    """
    returns True if filename is ignored when calculating hashing or archiving
    """
    return filename in [self.HASH_SUM_FILE, self.ARCHIVE_NAME] or \
           filename.endswith(self.PYC_EXT)

  def dbg_out(self, text):
    if self.DEBUG:
      sys.stderr.write("{0}\n".format(text))
    if not self.DEBUG and self.verbose:
      print text

def main(argv=None):
  """
  This method is called by maven during rpm creation.
  Params:
    1: Path to resources root directory
  """
  res_path = argv[1]

  if len(argv) >= 3:
    stacks_path = argv[2]
  else:
    stacks_path = os.path.join(res_path, ResourceFilesKeeper.STACKS_DIR)

  resource_files_keeper = ResourceFilesKeeper(res_path, stacks_path, nozip=True)
  resource_files_keeper.perform_housekeeping()

if __name__ == '__main__':
  main(sys.argv)

<commit_msg>AMBARI-20512: While generating hash for ambari-server to fill agent cache, all the symlinks should be traversed too (Madhuvanthi Radhakrishnan via jluniya)<commit_after>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import hashlib

import os, sys
import zipfile
import glob
import pprint

class KeeperException(Exception):
  pass

class ResourceFilesKeeper():
  """
  This class encapsulates all utility methods for resource files maintenance.
  """

  HOOKS_DIR="hooks"
  PACKAGE_DIR="package"
  STACKS_DIR="stacks"
  COMMON_SERVICES_DIR="common-services"
  CUSTOM_ACTIONS_DIR="custom_actions"
  HOST_SCRIPTS_DIR="host_scripts"
  DASHBOARDS_DIR="dashboards"
  EXTENSIONS_DIR="extensions"

  # For these directories archives are created
  ARCHIVABLE_DIRS = [HOOKS_DIR, PACKAGE_DIR]

  HASH_SUM_FILE=".hash"
  ARCHIVE_NAME="archive.zip"

  PYC_EXT=".pyc"
  METAINFO_XML = "metainfo.xml"

  BUFFER = 1024 * 32

  # Change that to True to see debug output at stderr
  DEBUG=False

  def __init__(self, resources_dir, stacks_dir, verbose=False, nozip=False):
    """
      nozip = create only hash files and skip creating zip archives
    """
    self.resources_dir = resources_dir
    self.stacks_root = stacks_dir
    self.verbose = verbose
    self.nozip = nozip

  def perform_housekeeping(self):
    """
    Performs housekeeping operations on resource files
    """
    self.update_directory_archives()
    # probably, later we will need some additional operations

  def _iter_update_directory_archive(self, subdirs_list):
    for subdir in subdirs_list:
      for root, dirs, _ in os.walk(subdir, followlinks=True):
        for d in dirs:
          if d in self.ARCHIVABLE_DIRS:
            full_path = os.path.abspath(os.path.join(root, d))
            self.update_directory_archive(full_path)

  def _update_resources_subdir_archive(self, subdir):
    archive_root = os.path.join(self.resources_dir, subdir)
    self.dbg_out("Updating archive for {0} dir at {1}...".format(subdir, archive_root))

    # update the directories so that the .hash is generated
    self.update_directory_archive(archive_root)

  def update_directory_archives(self):
    """
    Please see AMBARI-4481 for more details
    """
    # archive stacks
    self.dbg_out("Updating archives for stack dirs at {0}...".format(self.stacks_root))
    valid_stacks = self.list_stacks(self.stacks_root)
    self.dbg_out("Stacks: {0}".format(pprint.pformat(valid_stacks)))
    # Iterate over stack directories
    self._iter_update_directory_archive(valid_stacks)

    # archive common services
    common_services_root = os.path.join(self.resources_dir, self.COMMON_SERVICES_DIR)
    self.dbg_out("Updating archives for common services dirs at {0}...".format(common_services_root))
    valid_common_services = self.list_common_services(common_services_root)
    self.dbg_out("Common Services: {0}".format(pprint.pformat(valid_common_services)))
    # Iterate over common services directories
    self._iter_update_directory_archive(valid_common_services)

    # archive extensions
    extensions_root = os.path.join(self.resources_dir, self.EXTENSIONS_DIR)
    self.dbg_out("Updating archives for extensions dirs at {0}...".format(extensions_root))
    valid_extensions = self.list_extensions(extensions_root)
    self.dbg_out("Extensions: {0}".format(pprint.pformat(valid_extensions)))
    # Iterate over extension directories
    self._iter_update_directory_archive(valid_extensions)

    # custom actions
    self._update_resources_subdir_archive(self.CUSTOM_ACTIONS_DIR)

    # agent host scripts
    self._update_resources_subdir_archive(self.HOST_SCRIPTS_DIR)

    # custom service dashboards
    self._update_resources_subdir_archive(self.DASHBOARDS_DIR)

  def _list_metainfo_dirs(self, root_dir):
    valid_items = []  # Format: <stack_dir, ignore(True|False)>
    glob_pattern = "{0}/*/*".format(root_dir)
    dirs = glob.glob(glob_pattern)
    for directory in dirs:
      metainfo_file = os.path.join(directory, self.METAINFO_XML)
      if os.path.exists(metainfo_file):
        valid_items.append(directory)
    return valid_items

  def list_stacks(self, root_dir):
    """
    Builds a list of stack directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list stacks: {0}".format(str(err)))

  def list_common_services(self, root_dir):
    """
    Builds a list of common services directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list common services: {0}".format(str(err)))

  def list_extensions(self, root_dir):
    """
    Builds a list of extension directories
    """
    try:
      return self._list_metainfo_dirs(root_dir)
    except Exception, err:
      raise KeeperException("Can not list extensions: {0}".format(str(err)))

  def update_directory_archive(self, directory):
    """
    If hash sum for directory is not present or differs from saved value,
    recalculates hash sum and creates directory archive
    """
    skip_empty_directory = True
    cur_hash = self.count_hash_sum(directory)
    saved_hash = self.read_hash_sum(directory)
    if cur_hash != saved_hash:
      if not self.nozip:
        self.zip_directory(directory, skip_empty_directory)
      # Skip generation of .hash file is directory is empty
      if (skip_empty_directory and (not os.path.exists(directory) or not os.listdir(directory))):
        self.dbg_out("Empty directory. Skipping generation of hash file for {0}".format(directory))
      else:
        self.write_hash_sum(directory, cur_hash)
      pass

  def count_hash_sum(self, directory):
    """
    Recursively counts hash sum of all files in directory and subdirectories.
    Files and directories are processed in alphabetical order.
    Ignores previously created directory archives and files containing
    previously calculated hashes. Compiled pyc files are also ignored
    """
    try:
      sha1 = hashlib.sha1()
      file_list = []
      for root, dirs, files in os.walk(directory):
        for f in files:
          if not self.is_ignored(f):
            full_path = os.path.abspath(os.path.join(root, f))
            file_list.append(full_path)
      file_list.sort()
      for path in file_list:
        self.dbg_out("Counting hash of {0}".format(path))
        with open(path, 'rb') as fh:
          while True:
            data = fh.read(self.BUFFER)
            if not data:
              break
            sha1.update(data)
      return sha1.hexdigest()
    except Exception, err:
      raise KeeperException("Can not calculate directory "
                            "hash: {0}".format(str(err)))

  def read_hash_sum(self, directory):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    if os.path.isfile(hash_file):
      try:
        with open(hash_file) as fh:
          return fh.readline().strip()
      except Exception, err:
        raise KeeperException("Can not read file {0} : {1}".format(hash_file,
                                                                   str(err)))
    else:
      return None

  def write_hash_sum(self, directory, new_hash):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file, "w") as fh:
        fh.write(new_hash)
      os.chmod(hash_file, 0o755)
    except Exception, err:
      raise KeeperException("Can not write to file {0} : {1}".format(hash_file,
                                                                   str(err)))

  def zip_directory(self, directory, skip_if_empty = False):
    """
    Packs entire directory into zip file. Hash file is also packaged
    into archive
    """
    self.dbg_out("creating archive for directory {0}".format(directory))
    try:
      if skip_if_empty:
        if not os.path.exists(directory) or not os.listdir(directory):
          self.dbg_out("Empty directory. Skipping archive creation for {0}".format(directory))
          return

      zip_file_path = os.path.join(directory, self.ARCHIVE_NAME)
      zf = zipfile.ZipFile(zip_file_path, "w")
      abs_src = os.path.abspath(directory)
      for root, dirs, files in os.walk(directory):
        for filename in files:
          # Avoid zipping previous archive and hash file and binary pyc files
          if not self.is_ignored(filename):
            absname = os.path.abspath(os.path.join(root, filename))
            arcname = absname[len(abs_src) + 1:]
            self.dbg_out('zipping %s as %s' % (os.path.join(root, filename),
                                        arcname))
            zf.write(absname, arcname)
      zf.close()
      os.chmod(zip_file_path, 0o755)
    except Exception, err:
      raise KeeperException("Can not create zip archive of "
                            "directory {0} : {1}".format(directory, str(err)))

  def is_ignored(self, filename):
    """
    returns True if filename is ignored when calculating hashing or archiving
    """
    return filename in [self.HASH_SUM_FILE, self.ARCHIVE_NAME] or \
           filename.endswith(self.PYC_EXT)

  def dbg_out(self, text):
    if self.DEBUG:
      sys.stderr.write("{0}\n".format(text))
    if not self.DEBUG and self.verbose:
      print text

def main(argv=None):
  """
  This method is called by maven during rpm creation.
  Params:
    1: Path to resources root directory
  """
  res_path = argv[1]

  if len(argv) >= 3:
    stacks_path = argv[2]
  else:
    stacks_path = os.path.join(res_path, ResourceFilesKeeper.STACKS_DIR)

  resource_files_keeper = ResourceFilesKeeper(res_path, stacks_path, nozip=True)
  resource_files_keeper.perform_housekeeping()

if __name__ == '__main__':
  main(sys.argv)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# GNU General Public Licence (GPL)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA  02111-1307  USA
#

import optparse
import re
import time

def start_scann(module):
    # Open and read file
    method_list = []

    with open("../../ipol_demo/modules/{}/{}.py".format(module, module)) as f:
        lines = f.readlines()

    for line_index in range(len(lines)):

        method = ""
        if "@cherrypy.expose" in lines[line_index]:

            line_index += 1
            while not "def" in lines[line_index]:
                line_index += 1

            # Name of the method
            method += lines[line_index].strip() +"\n"
            while not "):" in lines[line_index]:
                line_index += 1
                method += lines[line_index][:-1] + "\n"

            line_index += 1
            # Check for comments or empty lines before docstring
            while lines[line_index].strip().startswith("#") or re.match(r'^\s*$', lines[line_index]):
                line_index += 1

            # Check if the method have any docstring
            if not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                method += "    \\error{this method does not have any docstring}\n"
            else:
                line_index += 1
                # Docstring
                while not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                    method += "    " + lines[line_index].strip() + "\n"
                    line_index += 1
            method += "\n"
            method_list.append(method)

    return {module:method_list}

def print_latex(content):
    result =""
    for module in content:
        result += "\\subsection{"+module.capitalize()+"} \n"
        result += "\\begin{itemize} \n"
        for method in content[module]:
            result += "\item {}".format(method.replace("_","\\_")
                                        .replace("\n","\n\\\\")
                                        .replace("<", "\textless")
                                        .replace(">", "\textgreater")
                                        .replace("&", "\&")
                                        .replace("{", "\{")
                                        .replace("}", "\}"))
        result += "\\end{itemize} \n"
    print result.replace("\\\\\\","\\")

# Parse program arguments
parser = optparse.OptionParser()
(opts, args) = parser.parse_args()

if len(args) > 1:
    print "Wrong number of arguments"
    exit(0)

module = args[0].lower() if len(args) == 1 else None
acepted_modules = ["core", "dispatcher", "demorunner", "demoinfo", "blobs", "archive", "conversion"]
date = time.strftime("%d/%m/%Y")
modules = []

if module in acepted_modules:
    modules.append(module)
elif module is None:
    modules = acepted_modules
else:
    print "Unknown module '{}'".format(module)
    exit(-1)

content = {}
for module in modules:
    content.update(start_scann(module))
print_latex(content)
<commit_msg>[doc/api] docstring_scanner.py converted to Python3<commit_after>#!/usr/bin/env python3
# -*- coding:utf-8 -*-

# GNU General Public Licence (GPL)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA  02111-1307  USA
#

import optparse
import re
import time
from sys import exit

def start_scann(module):
    # Open and read file
    method_list = []

    with open("../../ipol_demo/modules/{}/{}.py".format(module, module)) as f:
        lines = f.readlines()

    for line_index in range(len(lines)):

        method = ""
        if "@cherrypy.expose" in lines[line_index]:

            line_index += 1
            while not "def" in lines[line_index]:
                line_index += 1

            # Name of the method
            method += lines[line_index].strip() +"\n"
            while not "):" in lines[line_index]:
                line_index += 1
                method += lines[line_index][:-1] + "\n"

            line_index += 1
            # Check for comments or empty lines before docstring
            while lines[line_index].strip().startswith("#") or re.match(r'^\s*$', lines[line_index]):
                line_index += 1

            # Check if the method have any docstring
            if not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                method += "    \\error{this method does not have any docstring}\n"
            else:
                line_index += 1
                # Docstring
                while not ("\"\"\"" in lines[line_index] or "\'\'\'" in lines[line_index]):
                    method += "    " + lines[line_index].strip() + "\n"
                    line_index += 1
            method += "\n"
            method_list.append(method)

    return {module:method_list}

def print_latex(content):
    result =""
    for module in content:
        result += "\\subsection{"+module.capitalize()+"} \n"
        result += "\\begin{itemize} \n"
        for method in content[module]:
            result += "\item {}".format(method.replace("_","\\_")
                                        .replace("\n","\n\\\\")
                                        .replace("<", "\textless")
                                        .replace(">", "\textgreater")
                                        .replace("&", "\&")
                                        .replace("{", "\{")
                                        .replace("}", "\}"))
        result += "\\end{itemize} \n"
    print(result.replace("\\\\\\","\\"))

# Parse program arguments
parser = optparse.OptionParser()
(opts, args) = parser.parse_args()

if len(args) > 1:
    print("Wrong number of arguments")
    exit(-1)

module = args[0].lower() if len(args) == 1 else None
acepted_modules = ["core", "dispatcher", "demorunner", "demoinfo", "blobs", "archive", "conversion"]
date = time.strftime("%d/%m/%Y")
modules = []

if module in acepted_modules:
    modules.append(module)
elif module is None:
    modules = acepted_modules
else:
    print(f"Unknown module '{module}'")
    exit(-1)

content = {}
for module in modules:
    content.update(start_scann(module))
print_latex(content)
<|endoftext|>


<commit_before>import happybase
import datetime
import pprint

# collection of exception handlers and logging wrappers
import wrappers

class HBaseManager:

    def __init__(self):
        self.init_hbase_conn()
        self.establish_data_tables()
        self.init_table_handles()

    @wrappers.general_function_handler
    def init_hbase_conn(self):
        self.conn = happybase.Connection('localhost')

    @wrappers.general_function_handler
    def init_table_handles(self):
        self.state_stats_table = self.conn.table('state_stats')
        self.city_stats_table = self.conn.table('city_stats')
        self.county_stats_table = self.conn.table('county_stats')
        self.zipcode_stats_table = self.conn.table('zipcode_stats')

    @wrappers.general_function_handler
    def establish_data_tables(self):

        # parameters for all hbase tables
        # TODO put this schema in configuration

        families = {'cf':dict(max_versions=1)}

        # each in try-catch block because happybase api does not
        # have establish table, and throws exception
        try:
            self.conn.create_table('state_stats',families)
        except:
            pass

        try:
            self.conn.create_table('city_stats',families)
        except:
            pass

        try:
            self.conn.create_table('county_stats',families)
        except:
            pass

        try:
            self.conn.create_table('zipcode_stats',families)
        except:
            pass

    @wrappers.general_function_handler
    def get_list_volume(self, geo_type, geo_label, num_bedrooms, start_date, end_date):

        if geo_type == 'ST':
            table = self.state_stats_table
        elif geo_type == 'CI':
            table = self.city_stats_table
        elif geo_type == 'CO':
            table = self.county_stats_table
        elif geo_type == 'ZP':
            table = self.county_stats_table

        row_key = '|'.join([str(num_bedrooms), geo_label])

        data =  table.row(row_key)

        start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_datetime = datetime.datetime.strptime(end_date, '%Y-%m-%d')

        filtered_keys = [k for k in data.keys() if k.startswith('cf:') \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") < end_datetime \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") > start_datetime]

        sum_ = 0
        for key in filtered_keys:
            sum_ += int(eval(data[key])['n'])

        return sum_

    @wrappers.general_function_handler
    def get_list_average(self, state_code, city, num_bedrooms, start_date, end_date):

        if geo_type == 'ST':
            table = self.state_stats_table
        elif geo_type == 'CI':
            table = self.city_stats_table
        elif geo_type == 'CO':
            table = self.county_stats_table
        elif geo_type == 'ZP':
            table = self.county_stats_table

        row_key = '|'.join([str(num_bedrooms), geo_label])

        data = table.row(row_key)

        start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_datetime = datetime.datetime.strptime(end_date, '%Y-%m-%d')

        # Pythonic way to extract relevant columns of row
        filtered_keys = [k for k in data.keys() if k.startswith('cf:') \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") < end_datetime \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") > start_datetime]

        num = 0
        denom = 0
        for key in filtered_keys:
            num += int(eval(data[key])['a'])*int(eval(data[key])['n'])
            denom += int(eval(data[key])['n'])
        if denom == 0:
            return 0
        else:
            return round(float(num)/float(denom))

    @wrappers.general_function_handler
    def insert_json_doc_records(self, json_doc):

        if json_doc['doc_type'] == 'state_record':
            geo_label = json_doc['state_code']
            table = self.state_stats_table
        elif json_doc['doc_type'] == 'county_record':
            geo_label = json_doc['state_code'] + '-' + '_'.join(json_doc['county'].split(' '))
            table = self.county_stats_table
        elif json_doc['doc_type'] == 'zipcode_record':
            geo_label = json_doc['zipcode']
            table = self.zipcode_stats_table
        elif json_doc['doc_type'] == 'city_record':
            geo_label = json_doc['state_code'] + '-' + '_'.join(json_doc['city'].split(' '))
            table = self.city_stats_table

        geo_label = geo_label.lower()

        for bed_i in json_doc['stats']:
            row_dict = {}
            for week_i in json_doc['stats'][bed_i]:
                rec = json_doc['stats'][bed_i][week_i]
                row_dict['cf:' + week_i] = "'{'a':" + str(rec['a']) + ",'n':" + str(rec['n']) + "}'"

            row_key = str(bed_i) + '|' + geo_label
            table.put(row_key, row_dict)

def main():

    # TODO add config
    hm = HBaseManager()

if '__main__' == __name__:
    main()

<commit_msg>Extra ' characters around {} for the values on insert caused eval to fail on reads.<commit_after>import happybase
import datetime
import pprint

# collection of exception handlers and logging wrappers
import wrappers

class HBaseManager:

    def __init__(self):
        self.init_hbase_conn()
        self.establish_data_tables()
        self.init_table_handles()

    @wrappers.general_function_handler
    def init_hbase_conn(self):
        self.conn = happybase.Connection('localhost')

    @wrappers.general_function_handler
    def init_table_handles(self):
        self.state_stats_table = self.conn.table('state_stats')
        self.city_stats_table = self.conn.table('city_stats')
        self.county_stats_table = self.conn.table('county_stats')
        self.zipcode_stats_table = self.conn.table('zipcode_stats')

    @wrappers.general_function_handler
    def establish_data_tables(self):

        # parameters for all hbase tables
        # TODO put this schema in configuration

        families = {'cf':dict(max_versions=1)}

        # each in try-catch block because happybase api does not
        # have establish table, and throws exception
        try:
            self.conn.create_table('state_stats',families)
        except:
            pass

        try:
            self.conn.create_table('city_stats',families)
        except:
            pass

        try:
            self.conn.create_table('county_stats',families)
        except:
            pass

        try:
            self.conn.create_table('zipcode_stats',families)
        except:
            pass

    @wrappers.general_function_handler
    def get_list_volume(self, geo_type, geo_label, num_bedrooms, start_date, end_date):

        if geo_type == 'ST':
            table = self.state_stats_table
        elif geo_type == 'CI':
            table = self.city_stats_table
        elif geo_type == 'CO':
            table = self.county_stats_table
        elif geo_type == 'ZP':
            table = self.county_stats_table

        row_key = '|'.join([str(num_bedrooms), geo_label])

        data =  table.row(row_key)

        start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_datetime = datetime.datetime.strptime(end_date, '%Y-%m-%d')

        filtered_keys = [k for k in data.keys() if k.startswith('cf:') \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") < end_datetime \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") > start_datetime]

        sum_ = 0
        for key in filtered_keys:
            sum_ += int(eval(data[key])['n'])

        return sum_

    @wrappers.general_function_handler
    def get_list_average(self, state_code, city, num_bedrooms, start_date, end_date):

        if geo_type == 'ST':
            table = self.state_stats_table
        elif geo_type == 'CI':
            table = self.city_stats_table
        elif geo_type == 'CO':
            table = self.county_stats_table
        elif geo_type == 'ZP':
            table = self.county_stats_table

        row_key = '|'.join([str(num_bedrooms), geo_label])

        data = table.row(row_key)

        start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_datetime = datetime.datetime.strptime(end_date, '%Y-%m-%d')

        # Pythonic way to extract relevant columns of row
        filtered_keys = [k for k in data.keys() if k.startswith('cf:') \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") < end_datetime \
                             and datetime.datetime.strptime(k[3:],"%Y-%m-%d") > start_datetime]

        num = 0
        denom = 0
        for key in filtered_keys:
            num += int(eval(data[key])['a'])*int(eval(data[key])['n'])
            denom += int(eval(data[key])['n'])
        if denom == 0:
            return 0
        else:
            return round(float(num)/float(denom))

    @wrappers.general_function_handler
    def insert_json_doc_records(self, json_doc):

        if json_doc['doc_type'] == 'state_record':
            geo_label = json_doc['state_code']
            table = self.state_stats_table
        elif json_doc['doc_type'] == 'county_record':
            geo_label = json_doc['state_code'] + '-' + '_'.join(json_doc['county'].split(' '))
            table = self.county_stats_table
        elif json_doc['doc_type'] == 'zipcode_record':
            geo_label = json_doc['zipcode']
            table = self.zipcode_stats_table
        elif json_doc['doc_type'] == 'city_record':
            geo_label = json_doc['state_code'] + '-' + '_'.join(json_doc['city'].split(' '))
            table = self.city_stats_table

        geo_label = geo_label.lower()

        for bed_i in json_doc['stats']:
            row_dict = {}
            for week_i in json_doc['stats'][bed_i]:
                rec = json_doc['stats'][bed_i][week_i]
                row_dict['cf:' + week_i] = "{'a':" + str(rec['a']) + ",'n':" + str(rec['n']) + "}"

            row_key = str(bed_i) + '|' + geo_label
            table.put(row_key, row_dict)

def main():

    # TODO add config
    hm = HBaseManager()

if '__main__' == __name__:
    main()

<|endoftext|>


<commit_before>#! /usr/bin/env python
from time import strftime
from datetime import date, timedelta
import argparse
import sys
import tarfile
import hashlib
import os
import urllib2
from collections import defaultdict

# This tool builds Rust package manifests in the v2 (.toml) format. Manifests
# are for tools to use when finding and installing Rust and its various
# optional all_metadata, such as cross-compile-compatible variants of the
# standard library.

# Takes s3_addy, remote_dist_dir, component, channel.
# Emits a full v2 manifest to stdout.

# m     component
# n     channel
# r     remote_dist_dir
# s     s3_addy

def autoviv():
    # thanks https://en.wikipedia.org/wiki/Autovivification
    return defaultdict(autoviv)

def debug(words):
    # print words
    pass

def get_arguments():
    global component
    global channel
    global url_base
    global remote_dist_dir
    global listdir
   # Extract arguments from argv
    parser = argparse.ArgumentParser(description='Read inputs')
    parser.add_argument('-l','--listdir',
                        help="examples: /home/build/master/artefacts/")
    parser.add_argument('-m','--component',
                        help="examples: rust-docs, rustc, cargo")
    parser.add_argument('-n','--channel',
                        help="examples: beta, stable, nightly")
    parser.add_argument('-r','--remote_dist_dir',
                        help="example: dist")
    parser.add_argument('-s','--s3_addy',
                        help="example: s3://static-rust-lang-org")
    args = vars(parser.parse_args())
    component = args['component']
    channel = args['channel']
    remote_dist_dir = args['remote_dist_dir']
    if args['listdir']:
        listdir = args['listdir']
        if not listdir.endswith('/'):
            listdir += '/'
    else:
        listdir = '.'
    if  args['s3_addy'] and "dev-static-rust-lang-org" in args['s3_addy']:
        url_base = "https://dev-static.rust-lang.org"
    else:
        url_base = "https://static.rust-lang.org"

def print_preamble():
    # A manifest will always start with the version and date.
    print 'manifest_version = "2"'
    print 'date = "%s"' % strftime("%Y-%m-%d")

def build_metadata():
    global all_metadata
    files = [f for f in os.listdir(listdir) if os.path.isfile(listdir + f)]
    archives = [f for f in files if f.endswith('.tar.gz')]
    all_metadata = autoviv()
    for a in archives:
        d = decompose_name(a, channel)
        # d will return None if the archive is not in the channel we want
        if d:
            # d contains (triple, component)
            this_comp = d[1]
            triple = d[0]
            shasum = ''
            with open(listdir + a) as s:
                h = hashlib.sha256()
                h.update(s.read())
                shasum = h.hexdigest()
            (version, comp_list) = read_archive(listdir + a)
            all_metadata[this_comp]['version'] = version
            all_metadata[this_comp]['components'] = comp_list
            # FIXME: Assumption that this script runs on same day as artifacts
            # are placed. Worst case, this could be a day EARLIER, since
            # script output gets uploaded along with other artifacts
            url = url_base + '/' + remote_dist_dir + '/' + strftime("%Y-%m-%d") + '/' + a
            all_metadata[this_comp]['triples'][triple] = {'url': url,'hash': shasum, 'filename': a}

def decompose_name(filename, channel):
    # ASSUMPTIONS:
    #   * component names do not occur in triples
    #   * the filename ends with .tar.gz
    #   * Things are mostly hyphen-separated
    #   * We only want packages where the channel occurrs in the filename

    #   rust-docs   -   nightly   -   i686-apple-darwin    .tar.gz
    #    \    /            |              \    /              |
    #   component       channel           triple          extension
    component = None
    triple = None
    valid_components = [
                        "cargo",
                        "rust-docs",
                        "rust-mingw",
                        "rust-std",
                        "rustc",
                        "rust",
                        ]
    debug("decomposing " + filename)
    # Strip extension. TODO: handle non-tgz
    if filename.endswith(".tar.gz"):
        filename = filename[:-7]
    if channel not in filename:
        return
    # still here? filename looks like rust-docs--i686-apple-darwin
    for c in valid_components:
        if c in filename:
            component = c
    for t in all_triples:
        if t in filename:
            triple = t
    if 'src' in filename:
        triple = 'src'
    if triple and component:
        return (triple, component)

def read_archive(a):
    version = ''
    comp_list = []
    ar = tarfile.open(a, "r:gz")
    listing = ar.getmembers()
    for l in listing:
        path = l.name.split('/')
        if len(path) >= 2:
            if path[1] == 'version':
                f = ar.extractfile(l)
                version = f.read().strip()
                #debug("version: " + str(version))
            if path[1] == 'components':
                f = ar.extractfile(l)
                comp_list = f.read().split()
                #debug("all_metadata: " + str(all_metadata))
    ar.close()
    return (version, comp_list)

def try_getting_cargo(when, triple):
    global all_metadata
    if (date.today() - when).days >= 62:
        # there hasn't been a Cargo for this triple in over 2 months...
        # Odds that Cargo older than that will do the right thing are low.
        return
    tryurl = url_base + "/cargo-dist/" + when.strftime("%Y-%m-%d")
    tryurl += "/cargo-" + channel + '-' + triple + '.tar.gz.sha256'
    try:
        response = urllib2.urlopen(tryurl)
        shahash = response.read().split()[0]
        all_metadata['cargo']['triples'][triple]['url'] = tryurl[:-7]
        all_metadata['cargo']['triples'][triple]['hash'] = shahash
    #FIXME need more sophisticated error handling
    except urllib2.HTTPError as e:
        # server's busted?
        pass
    except urllib2.URLError as e:
        # There wasn't a Cargo on that date. Try the day before.
        try_getting_cargo(when - timedelta(days=1), triple)

def get_cargo():
    if isinstance(all_metadata['cargo']['version'], basestring):
        # We have a cargo from this build. Where'd that come from?
        return
    # Cargo is built daily and dumped into baseurl/cargo-dist/
    for t in cargo_triples:
        try_getting_cargo(date.today(), t)

def print_rust_metadata():
    global rust_version
    c = all_metadata['rust']
    print "[pkg.rust]"
    rust_version = c['version']
    if not isinstance(rust_version, basestring):
        e = "No rust-" + channel + "-*.tgz packages were found in " + listdir
        raise Exception(e)
    print '    version = "%s"' % rust_version
    if 'src' in c['triples']:
        print "    [pkg.%s.src]" % ('rust')
        print '        url = "%s"' % c['triples']['src']['url']
        print '        hash = "%s"' % c['triples']['src']['hash']
        del c['triples']['src']

    exts = []
    for t in sorted(c['triples']):
        # Each triple has url & hash, components each with pkg and target,
        # extensions each with pkg & target, and later installers each with
        # type, url, and hash.
        print "    [pkg.%s.target.%s]" % ('rust', t)
        print '        url = "%s"' % c['triples'][t]['url']
        print '        hash = "%s"' % c['triples'][t]['hash']
        for comp in sorted(c['components']):
            if t in all_metadata[comp]['triples']:
                # we have the version of the component that matches the
                # triple. This handles the matching std as a component, as
                # well.
                print "        [[pkg.%s.target.%s.components]]" % (component, t)
                print '            pkg = "%s"' % comp
                print '            target = "%s"' % t
            elif comp == 'std':
                # this is a std for some other triple. It's an extension.
                exts.append('        [[pkg.%s.target.%s.extensions]]' % (component, t))
                exts.append('            pkg = "%s"' % comp)
                exts.append('            target = "%s"' % trip)
            elif comp != 'cargo' and comp != 'rust-docs':
                e = "Component " + comp + ' - ' + channel + ' - ' + t + " needed but not found"
                raise Exception(e)

    for e in exts:
            print e

def print_component_metadata(c):
    print "[pkg.%s]" % c
    comp_version = all_metadata[c]['version']
    if not isinstance(comp_version, basestring):
        comp_version = rust_version
    if len(comp_version) <= 3:
        comp_version = rust_version
    print '    version = "%s"' % comp_version
    trips = all_metadata[c]['triples']
    if 'src' in trips:
        print "    [pkg.%s.src]" % c
        print '        url = "%s"' % trips['src']['url']
        print '        hash = "%s"' % trips['src']['hash']
    for possibility in all_triples:
        print '    [pkg.%s.target.%s]' % (c, possibility)
        if possibility in trips:
            print '        available = true'
            print '        url = "%s"' % trips[possibility]['url']
            print '        hash = "%s"' % trips[possibility]['hash']
        else:
            print '        available = false'

def main():
    # Not every component (docs, etc.) carries around the rust version string.
    # This global holds the version string for rust proper so it can be filled in
    # on those all_metadata which are missing it.
    global rust_version
    global all_triples
    global cargo_triples
    all_triples = [
                    "aarch64-unknown-linux-gnu",
                    "arm-linux-androideabi",
                    "arm-unknown-linux-gnueabif",
                    "arm-unknown-linux-gnueabihf",
                    "i686-apple-darwin",
                    "i686-pc-windows-gnu",
                    "i686-pc-windows-msvc",
                    "i686-unknown-linux-gnu",
                    "mips-unknown-linux",
                    "mipsel-unknown-linux",
                    "x86_64-apple-darwin",
                    "x86_64-pc-windows-gnu",
                    "x86_64-pc-windows-msvc",
                    "x86_64-unknown-linux-gnu",
                    "x86_64-unknown-linux-musl",
                    ]
    get_arguments()
    print_preamble()
    build_metadata()
    cargo_triples = all_triples
    get_cargo() # Make a better effort to get ahold of some Cargo package info
    # FIXME: Maybe don't assume we always have Rust? But we probably always
    # have Rust, and its metadata is quite different from components.
    print_rust_metadata()
    for c in sorted(all_metadata):
        if c != 'rust':
            print_component_metadata(c)

if __name__ == "__main__":
    main()
<commit_msg>verboser names<commit_after>#! /usr/bin/env python
from time import strftime
from datetime import date, timedelta
import argparse
import sys
import tarfile
import hashlib
import os
import urllib2
from collections import defaultdict

# This tool builds Rust package manifests in the v2 (.toml) format. Manifests
# are for tools to use when finding and installing Rust and its various
# optional all_metadata, such as cross-compile-compatible variants of the
# standard library.

# Takes s3_addy, remote_dist_dir, component, channel.
# Emits a full v2 manifest to stdout.

# m     component
# n     channel
# r     remote_dist_dir
# s     s3_addy

def autoviv():
    # thanks https://en.wikipedia.org/wiki/Autovivification
    return defaultdict(autoviv)

def debug(words):
    # print words
    pass

def get_arguments():
    global component
    global channel
    global url_base
    global remote_dist_dir
    global directory_to_list
   # Extract arguments from argv
    parser = argparse.ArgumentParser(description='Read inputs')
    parser.add_argument('-l','--directory_to_list',
                        help="examples: /home/build/master/artefacts/")
    parser.add_argument('-m','--component',
                        help="examples: rust-docs, rustc, cargo")
    parser.add_argument('-n','--channel',
                        help="examples: beta, stable, nightly")
    parser.add_argument('-r','--remote_dist_dir',
                        help="example: dist")
    parser.add_argument('-s','--s3_addy',
                        help="example: s3://static-rust-lang-org")
    args = vars(parser.parse_args())
    component = args['component']
    channel = args['channel']
    remote_dist_dir = args['remote_dist_dir']
    if args['directory_to_list']:
        directory_to_list = args['directory_to_list']
        if not directory_to_list.endswith('/'):
            directory_to_list += '/'
    else:
        directory_to_list = '.'
    if  args['s3_addy'] and "dev-static-rust-lang-org" in args['s3_addy']:
        url_base = "https://dev-static.rust-lang.org"
    else:
        url_base = "https://static.rust-lang.org"

def print_preamble():
    # A manifest will always start with the version and date.
    print 'manifest_version = "2"'
    print 'date = "%s"' % strftime("%Y-%m-%d")

def build_metadata():
    global all_metadata
    files = [f for f in os.listdir(directory_to_list) if os.path.isfile(directory_to_list + f)]
    archives = [f for f in files if f.endswith('.tar.gz')]
    all_metadata = autoviv()
    for a in archives:
        d = decompose_name(a, channel)
        # d will return None if the archive is not in the channel we want
        if d:
            # d contains (triple, component)
            this_component = d[1]
            triple = d[0]
            shasum = ''
            with open(directory_to_list + a) as s:
                h = hashlib.sha256()
                h.update(s.read())
                shasum = h.hexdigest()
            (version, comp_list) = read_archive(directory_to_list + a)
            all_metadata[this_component]['version'] = version
            all_metadata[this_component]['components'] = comp_list
            # FIXME: Assumption that this script runs on same day as artifacts
            # are placed. Worst case, this could be a day EARLIER, since
            # script output gets uploaded along with other artifacts
            url = url_base + '/' + remote_dist_dir + '/' + strftime("%Y-%m-%d") + '/' + a
            all_metadata[this_component]['triples'][triple] = {'url': url,'hash': shasum, 'filename': a}

def decompose_name(filename, channel):
    # ASSUMPTIONS:
    #   * component names do not occur in triples
    #   * the filename ends with .tar.gz
    #   * Things are mostly hyphen-separated
    #   * We only want packages where the channel occurrs in the filename

    #   rust-docs   -   nightly   -   i686-apple-darwin    .tar.gz
    #    \    /            |              \    /              |
    #   component       channel           triple          extension
    component = None
    triple = None
    valid_components = [
                        "cargo",
                        "rust-docs",
                        "rust-mingw",
                        "rust-std",
                        "rustc",
                        "rust",
                        ]
    debug("decomposing " + filename)
    # Strip extension. TODO: handle non-tgz
    if filename.endswith(".tar.gz"):
        filename = filename[:-7]
    if channel not in filename:
        return
    # still here? filename looks like rust-docs--i686-apple-darwin
    for c in valid_components:
        if c in filename:
            component = c
    for t in all_triples:
        if t in filename:
            triple = t
    if 'src' in filename:
        triple = 'src'
    if triple and component:
        return (triple, component)

def read_archive(a):
    version = ''
    comp_list = []
    ar = tarfile.open(a, "r:gz")
    listing = ar.getmembers()
    for l in listing:
        path = l.name.split('/')
        if len(path) >= 2:
            if path[1] == 'version':
                f = ar.extractfile(l)
                version = f.read().strip()
                #debug("version: " + str(version))
            if path[1] == 'components':
                f = ar.extractfile(l)
                comp_list = f.read().split()
                #debug("all_metadata: " + str(all_metadata))
    ar.close()
    return (version, comp_list)

def try_getting_cargo(when, triple):
    global all_metadata
    if (date.today() - when).days >= 62:
        # there hasn't been a Cargo for this triple in over 2 months...
        # Odds that Cargo older than that will do the right thing are low.
        return
    tryurl = url_base + "/cargo-dist/" + when.strftime("%Y-%m-%d")
    tryurl += "/cargo-" + channel + '-' + triple + '.tar.gz.sha256'
    try:
        response = urllib2.urlopen(tryurl)
        shahash = response.read().split()[0]
        all_metadata['cargo']['triples'][triple]['url'] = tryurl[:-7]
        all_metadata['cargo']['triples'][triple]['hash'] = shahash
    #FIXME need more sophisticated error handling
    except urllib2.HTTPError as e:
        # server's busted?
        pass
    except urllib2.URLError as e:
        # There wasn't a Cargo on that date. Try the day before.
        try_getting_cargo(when - timedelta(days=1), triple)

def get_cargo():
    if isinstance(all_metadata['cargo']['version'], basestring):
        # We have a cargo from this build. Where'd that come from?
        return
    # Cargo is built daily and dumped into baseurl/cargo-dist/
    for t in cargo_triples:
        try_getting_cargo(date.today(), t)

def print_rust_metadata():
    global rust_version
    c = all_metadata['rust']
    print "[pkg.rust]"
    rust_version = c['version']
    if not isinstance(rust_version, basestring):
        e = "No rust-" + channel + "-*.tgz packages were found in " + directory_to_list
        raise Exception(e)
    print '    version = "%s"' % rust_version
    if 'src' in c['triples']:
        print "    [pkg.%s.src]" % ('rust')
        print '        url = "%s"' % c['triples']['src']['url']
        print '        hash = "%s"' % c['triples']['src']['hash']
        del c['triples']['src']

    exts = []
    for t in sorted(c['triples']):
        # Each triple has url & hash, components each with pkg and target,
        # extensions each with pkg & target, and later installers each with
        # type, url, and hash.
        print "    [pkg.%s.target.%s]" % ('rust', t)
        print '        url = "%s"' % c['triples'][t]['url']
        print '        hash = "%s"' % c['triples'][t]['hash']
        for comp in sorted(c['components']):
            if t in all_metadata[comp]['triples']:
                # we have the version of the component that matches the
                # triple. This handles the matching std as a component, as
                # well.
                print "        [[pkg.%s.target.%s.components]]" % (component, t)
                print '            pkg = "%s"' % comp
                print '            target = "%s"' % t
            elif comp == 'std':
                # this is a std for some other triple. It's an extension.
                exts.append('        [[pkg.%s.target.%s.extensions]]' % (component, t))
                exts.append('            pkg = "%s"' % comp)
                exts.append('            target = "%s"' % trip)
            elif comp != 'cargo' and comp != 'rust-docs':
                e = "Component " + comp + ' - ' + channel + ' - ' + t + " needed but not found"
                raise Exception(e)

    for e in exts:
            print e

def print_component_metadata(c):
    print "[pkg.%s]" % c
    comp_version = all_metadata[c]['version']
    if not isinstance(comp_version, basestring):
        comp_version = rust_version
    if len(comp_version) <= 3:
        comp_version = rust_version
    print '    version = "%s"' % comp_version
    trips = all_metadata[c]['triples']
    if 'src' in trips:
        print "    [pkg.%s.src]" % c
        print '        url = "%s"' % trips['src']['url']
        print '        hash = "%s"' % trips['src']['hash']
    for possibility in all_triples:
        print '    [pkg.%s.target.%s]' % (c, possibility)
        if possibility in trips:
            print '        available = true'
            print '        url = "%s"' % trips[possibility]['url']
            print '        hash = "%s"' % trips[possibility]['hash']
        else:
            print '        available = false'

def main():
    # Not every component (docs, etc.) carries around the rust version string.
    # This global holds the version string for rust proper so it can be filled in
    # on those all_metadata which are missing it.
    global rust_version
    global all_triples
    global cargo_triples
    all_triples = [
                    "aarch64-unknown-linux-gnu",
                    "arm-linux-androideabi",
                    "arm-unknown-linux-gnueabif",
                    "arm-unknown-linux-gnueabihf",
                    "i686-apple-darwin",
                    "i686-pc-windows-gnu",
                    "i686-pc-windows-msvc",
                    "i686-unknown-linux-gnu",
                    "mips-unknown-linux",
                    "mipsel-unknown-linux",
                    "x86_64-apple-darwin",
                    "x86_64-pc-windows-gnu",
                    "x86_64-pc-windows-msvc",
                    "x86_64-unknown-linux-gnu",
                    "x86_64-unknown-linux-musl",
                    ]
    get_arguments()
    print_preamble()
    build_metadata()
    cargo_triples = all_triples
    get_cargo() # Make a better effort to get ahold of some Cargo package info
    # FIXME: Maybe don't assume we always have Rust? But we probably always
    # have Rust, and its metadata is quite different from components.
    print_rust_metadata()
    for c in sorted(all_metadata):
        if c != 'rust':
            print_component_metadata(c)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from operator import attrgetter

from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.db.models import Q
from django.forms import ModelForm, CharField, Textarea

from django_ace import AceWidget
from judge.models import Organization, Profile, Submission, Problem
from judge.widgets import MathJaxPagedownWidget, PagedownWidget

class ProfileForm(ModelForm):
    class Meta:
        model = Profile
        fields = ['name', 'about', 'organization', 'timezone', 'language', 'ace_theme']

class ProblemSubmitForm(ModelForm):
    source = CharField(max_length=65536, widget=AceWidget(theme='twilight'))

    def __init__(self, *args, **kwargs):
        super(ProblemSubmitForm, self).__init__(*args, **kwargs)
        self.fields['problem'].empty_label = None
        self.fields['problem'].widget = forms.HiddenInput()
        self.fields['language'].empty_label = None
        self.fields['language'].label_from_instance = attrgetter('display_name')

    class Meta:
        model = Submission
        fields = ['problem', 'source', 'language']

class OrganizationForm(ModelForm):
    class Meta:
        model = Organization
        fields = ['name', 'key', 'about']

class EditOrganizationForm(ModelForm):
    class Meta:
        model = Organization
        fields = ['name', 'short_name', 'about', 'admins']
        widgets = {
            'admins': FilteredSelectMultiple('Admins', False)
        }
        if PagedownWidget is not None:
            widgets['about'] = MathJaxPagedownWidget

class NewOrganizationForm(EditOrganizationForm):
    class Meta(EditOrganizationForm.Meta):
        fields = ['key'] + EditOrganizationForm.Meta.fields

class ProblemEditForm(ModelForm):
    class Meta:
        model = Problem
        fields = ['name', 'is_public', 'authors', 'types', 'group', 'description', 'time_limit',
                  'memory_limit', 'points', 'partial', 'allowed_languages']
        widgets = {
            'authors': FilteredSelectMultiple('creators', False),
            'types': FilteredSelectMultiple('types', False),
            'allowed_languages': FilteredSelectMultiple('languages', False),
        }
        if MathJaxPagedownWidget is not None:
            widgets['description'] = MathJaxPagedownWidget

    def __init__(self, *args, **kwargs):
        super(ProblemEditForm, self).__init__(*args, **kwargs)
        self.fields['authors'].queryset = Profile.objects.filter(Q(user__groups__name__in=['ProblemSetter', 'Admin']) |
                                                                 Q(user__is_superuser=True))

class ProblemAddForm(ProblemEditForm):
    class Meta(ProblemEditForm.Meta):
        fields = ['code'] + ProblemEditForm.Meta.fields
<commit_msg>Add new message form<commit_after>from operator import attrgetter

from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.db.models import Q
from django.forms import ModelForm, CharField, Textarea

from django_ace import AceWidget
from judge.models import Organization, Profile, Submission, Problem
from judge.widgets import MathJaxPagedownWidget, PagedownWidget

class ProfileForm(ModelForm):
    class Meta:
        model = Profile
        fields = ['name', 'about', 'organization', 'timezone', 'language', 'ace_theme']

class ProblemSubmitForm(ModelForm):
    source = CharField(max_length=65536, widget=AceWidget(theme='twilight'))

    def __init__(self, *args, **kwargs):
        super(ProblemSubmitForm, self).__init__(*args, **kwargs)
        self.fields['problem'].empty_label = None
        self.fields['problem'].widget = forms.HiddenInput()
        self.fields['language'].empty_label = None
        self.fields['language'].label_from_instance = attrgetter('display_name')

    class Meta:
        model = Submission
        fields = ['problem', 'source', 'language']

class OrganizationForm(ModelForm):
    class Meta:
        model = Organization
        fields = ['name', 'key', 'about']

class EditOrganizationForm(ModelForm):
    class Meta:
        model = Organization
        fields = ['name', 'short_name', 'about', 'admins']
        widgets = {
            'admins': FilteredSelectMultiple('Admins', False)
        }
        if PagedownWidget is not None:
            widgets['about'] = MathJaxPagedownWidget

class NewMessageForm(ModelForm):
    class Meta:
        model = Organization
        fields = ['title', 'content']
        widgets = {}
        if PagedownWidget is not None:
            widgets.update({'content', MathJaxPagedownWidget()})

class NewOrganizationForm(EditOrganizationForm):
    class Meta(EditOrganizationForm.Meta):
        fields = ['key'] + EditOrganizationForm.Meta.fields

class ProblemEditForm(ModelForm):
    class Meta:
        model = Problem
        fields = ['name', 'is_public', 'authors', 'types', 'group', 'description', 'time_limit',
                  'memory_limit', 'points', 'partial', 'allowed_languages']
        widgets = {
            'authors': FilteredSelectMultiple('creators', False),
            'types': FilteredSelectMultiple('types', False),
            'allowed_languages': FilteredSelectMultiple('languages', False),
        }
        if MathJaxPagedownWidget is not None:
            widgets['description'] = MathJaxPagedownWidget

    def __init__(self, *args, **kwargs):
        super(ProblemEditForm, self).__init__(*args, **kwargs)
        self.fields['authors'].queryset = Profile.objects.filter(Q(user__groups__name__in=['ProblemSetter', 'Admin']) |
                                                                 Q(user__is_superuser=True))

class ProblemAddForm(ProblemEditForm):
    class Meta(ProblemEditForm.Meta):
        fields = ['code'] + ProblemEditForm.Meta.fields
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# concat json or yaml files into large k8s List
# p.py file1 file2 file3
# produces all files on output wrapped by an enclosing list
import yaml
import sys

def run():
    av = sys.argv
    av.pop(0)

    s = { "apiVersion":"v1beta3", "kind":"List", "items":[] }

    for i in range(len(av)):
        s['items'].append(yaml.load(open(av[i]).read()))

    print yaml.dump(s)

if __name__ == '__main__':
   run()
<commit_msg>added arg parser, ability to output json or yaml, with json the output can be pretty, also input can come from stdin (-), finally, processing can be a single file or many files. the main reason i wrote this is because kube-version-change only parses a json file, all of my files are yaml. so the pipeline: jycat -t json file.yaml | kube-version-change | jycat.py > new.yaml can now be done to modify the version of a yaml file.<commit_after>#!/usr/bin/env python2.7
# concat json or yaml files into large k8s List
# p.py file1 file2 file3
# produces all files on output wrapped by an enclosing list

import yaml, json, sys, os, argparse, logging
from os.path import expanduser
import argparse_config

class SmartFormatter(argparse.HelpFormatter):

    def _split_lines(self, text, width):
        # this is the RawTextHelpFormatter._split_lines
        if text.startswith('R|'):
            return text[2:].splitlines()
        return argparse.HelpFormatter._split_lines(self, text, width)

def run():
    loglevel = "INFO"

    home = expanduser("~")
    prog = os.path.basename(__file__)
    cfn = home + '/' + '.' + os.path.splitext(prog)[0] + '.conf'

    p = argparse.ArgumentParser(description="yaml json k8s laundry",
            formatter_class=SmartFormatter)

    # overall app related stuff
    p.add_argument('-p', '--pretty', action='store_true', dest='pretty', default=False)
    p.add_argument('-t', '--type', action='store', dest='output_type',
            default='yaml',
            choices=['json','yaml'],
            help='Output type, json or yaml' )
    # these are the command line leftovers, the files to process
    p.add_argument('files', nargs='*')

    # non application related stuff
    p.add_argument('-l', '--loglevel', action='store', dest='loglevel',
            default=loglevel,
            choices=['DEBUG','INFO','WARNING','ERROR','CRITICAL'],
            help='Log level (DEBUG,INFO,WARNING,ERROR,CRITICAL) default is: '+loglevel)
    p.add_argument('-s', '--save', action='store_true', dest='save',
            default=False, help='save select command line arguments (default is always) in "'+cfn+'" file')

    # read in defaults from ~/.PROGBASENAMENOSUFFIX
    # if the file exists
    if os.path.isfile(cfn):
        argparse_config.read_config_file(p,cfn)

    # parse arguments (after reading defaults from ~/.dot file
    args = p.parse_args()
    if args.loglevel:
        loglevel = args.loglevel

    # set our logging level (from -l INFO (or whatever))
    numeric_level = getattr(logging, loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s', loglevel)
    logging.basicConfig(level=numeric_level)

    logging.info('Program starting :%s', prog)
    logging.debug('Arg: pretty     :%s', args.pretty)
    logging.debug('Arg: type       :%s', args.output_type)
    logging.debug('Arg: loglevel   :%s', loglevel)
    logging.debug('Arg: save       :%s', args.save)

    # save to the defaults file if a -s specified on command line
    if args.save:
        f = open(cfn, 'w')
        # remove the 'save' from the file
        f.write(re.sub('\naddminion\n', re.sub('\nsave\n','\n',argparse_config.generate_config(p, args, section='default'))))
        f.close()

    # here we go, start of program here
    s_out = { "apiVersion":"v1beta3", "kind":"List", "items":[] }

    av = args.files
    for i in range(len(av)):
        i_file = None
        if av[i] == '-':
            i_file = sys.stdin
        else:
            i_file = open(av[i])
        s_out['items'].append(yaml.load(i_file.read()))

    # if we are only messing with a single file...
    s = s_out
    if len(av) == 1:
        s = s_out['items'][0]

    if args.output_type == 'yaml':
        print yaml.dump(s)
    elif args.output_type == 'json':
        if args.pretty:
            print json.dumps(s, sort_keys=True, indent=4, separators=(',', ': '))
        else:
            print json.dumps(s)
    else:
        raise ValueError('Invalid output type : %s', args.output_type)

if __name__ == '__main__':
   run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Standard library imports
import sys

# Third party imports
from django.test import TestCase

# Local application / specific library imports
from precise_bbcode.parser import BBCodeParser

class ParserTestCase(TestCase):
    DEFAULT_TAGS_RENDERING_TESTS = (
        # BBcodes without errors
        ('[b]hello world![/b]', 'hello world!'),
        ('[b]hello [i]world![/i][/b]', 'hello world!'),
        ('[b]hello [ world![/b]', 'hello [ world!'),
        ('[b]]he[llo [ w]orld![/b]', ']he[llo [ w]orld!'),
        ('[ b ]hello [u]world![/u][ /b ]', 'hello world!'),
        ('[b]hello [] world![/b]', 'hello [] world!'),
        ('[list]\n[*]one\n[*]two\n[/list]', '

one

two

'),
        ('[list=1]\n[*]item 1\n[*]item 2\n[/list]', '

item 1

item 2

'),
        ('[list] [*]Item 1 [*]Item 2 [*]Item 3   [/list]', '

Item 1

Item 2

Item 3

'),
        ('>> some special chars >< <>', '>> some special chars >< <>'),
        ('"quoted text"', '"quoted text"'),
        ('>> some other special chars', '>> some other special chars'),
        ('[url]http://foo.com/bar.php?some--data[/url]', 'http://foo.com/bar.php?some--data'),
        ('[url]http://www.google.com[/url]', 'http://www.google.com'),
        ('[url=google.com]goto google[/url]', 'goto google'),
        ('[url=http://google.com][/url]', 'http://google.com'),
        ('[URL=google.com]goto google[/URL]', 'goto google'),
        ('[url=]xss[/url]', 'xss'),
        ('www.google.com foo.com/bar http://xyz.ci', 'www.google.com foo.com/bar http://xyz.ci'),
        ('[url=relative/foo/bar.html]link[/url]', 'link'),
        ('[url=/absolute/foo/bar.html]link[/url]', 'link'),
        ('[url=./hello.html]world![/url]', 'world!'),
        ('[img]http://www.foo.com/bar/img.png[/img]', ''),
        ('[quote] \r\nhello\nworld! [/quote]', '
hello
world!
'),
        ('[code][b]hello world![/b][/code]', '[b]hello world![/b]'),
        ('[color=green]goto [url=google.com]google website[/url][/color]', 'goto google website'),
        ('[color=#FFFFFF]white[/color]', 'white'),
        ('[color=]xss[/color]', '[color=<script></script>]xss[/color]'),
        ('[COLOR=blue]hello world![/color]', 'hello world!'),
        # BBCodes with syntactic errors
        ('[b]z sdf s s', '[b]z sdf s s'),
        ('[b][i]hello world![/b][/i]', '[i]hello world![/i]'),
        ('[b]hello [i]world![/i]', '[b]hello world!'),
        ('[color]test[/color]', '[color]test[/color]'),
        ('[/abcdef][/i]', '[/abcdef][/i]'),
        ('[b\n hello [i]the[/i] world![/b]', '[b
 hello the world![/b]'),
        ('[b]hello [i]the[/b] world![/i]', 'hello [i]the world![/i]'),
        ('[b] hello the[u]world ![/i] see you[/b]', ' hello the[u]world ![/i] see you'),
        # BBCodes with semantic errors
        ('[color=some words]test[/color]', '[color=some words]test[/color]'),
        # Unknown BBCodes
        ('[unknown][hello][/unknown]', '[unknown][hello][/unknown]'),
    )

    CUSTOM_TAGS_RENDERING_TESTS = {
        'tags': {
            'justify': {
                'args': ('justify', '[justify]{TEXT}[/justify]', '
{TEXT}
'),
                'kwargs': {},
            },
            'spoiler': {
                'args': ('spoiler', '[spoiler]{TEXT}[/spoiler]', '

    <input type="button" value="Afficher" style="width:60px;font-size:10px;margin:0px;padding:0px;" onclick="if (this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display != '') { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = '';        this.innerText = ''; this.value = \'Masquer\'; } else { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = \'none\'; this.innerText = ''; this.value = \'Afficher\'; }" />

{TEXT}

'),
                'kwargs': {},
            },
            'youtube': {
                'args': ('youtube', '[youtube]{TEXT}[/youtube]', '<object width="425" height="350"><param name="movie" value="http://www.youtube.com/v/{TEXT}"></param><param name="wmode" value="transparent"></param><embed src="http://www.youtube.com/v/{TEXT}" type="application/x-shockwave-flash" wmode="transparent" width="425" height="350"></embed></object>'),
                'kwargs': {},
            },
            'h1': {
                'args': ('h1', '[h1={COLOR}]{TEXT}[/h1]', '{TEXT}
'),
                'kwargs': {},
            },
            'hr': {
                'args': ('hr', '[hr]', '
'),
                'kwargs': {'standalone': True},
            },
            'size': {
                'args': ('size', '[size={NUMBER}]{TEXT}[/size]', '{TEXT}'),
                'kwargs': {},
            },
            'mailto': {
                'args': ('email', '[email]{EMAIL}[/email]', '{EMAIL}'),
                'kwargs': {'replace_links': False},
            },
            'simpletext': {
                'args': ('simpletext', '[simpletext]{SIMPLETEXT}[/simpletext]', '{SIMPLETEXT}'),
                'kwargs': {},
            }
        },
        'tests': (
            # BBcodes without errors
            ('[justify]hello world![/justify]', '
hello world!
'),
            ('[spoiler]hidden![/spoiler]', '

    <input type="button" value="Afficher" style="width:60px;font-size:10px;margin:0px;padding:0px;" onclick="if (this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display != '') { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = '';        this.innerText = ''; this.value = \'Masquer\'; } else { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = \'none\'; this.innerText = ''; this.value = \'Afficher\'; }" />

hidden!

'),
            ('[youtube]ztD3mRMdqSw[/youtube]', '<object width="425" height="350"><param name="movie" value="http://www.youtube.com/v/ztD3mRMdqSw"></param><param name="wmode" value="transparent"></param><embed src="http://www.youtube.com/v/ztD3mRMdqSw" type="application/x-shockwave-flash" wmode="transparent" width="425" height="350"></embed></object>'),
            ('[h1=#FFF]hello world![/h1]', 'hello world!
'),
            ('[hr]', '
'),
            ('[size=24]hello world![/size]', 'hello world!'),
            ('[email]xyz@xyz.com[/email]', 'xyz@xyz.com'),
            ('[email]xyz.fr@xyz.com[/email]', 'xyz.fr@xyz.com'),
            ('[simpletext]hello world[/simpletext]', 'hello world'),
            # BBCodes with semantic errors
            ('[size=]hello world![/size]', '[size]hello world![/size]'),
            ('[size=hello]hello world![/size]', '[size=hello]hello world![/size]'),
            ('[email]hello world![/email]', '[email]hello world![/email]'),
            ('[email]http://www.google.com[/email]', '[email]http://www.google.com[/email]'),
            ('[email=12]24[/email]', '[email]24[/email]'),
            ('[simpletext]hello world![/simpletext]', '[simpletext]hello world![/simpletext]'),
            ('[simpletext]hello #~ world[/simpletext]', '[simpletext]hello #~ world[/simpletext]'),
        )
    }

    def setUp(self):
        self.parser = BBCodeParser()

    def test_default_tags_rendering(self):
        # Run & check
        for bbcodes_text, expected_html_text in self.DEFAULT_TAGS_RENDERING_TESTS:
            result = self.parser.render(bbcodes_text)
            self.assertEqual(result, expected_html_text)

    def test_custom_tags_rendering(self):
        # Setup
        for _, tag_def in self.CUSTOM_TAGS_RENDERING_TESTS['tags'].items():
            self.parser.add_default_renderer(*tag_def['args'], **tag_def['kwargs'])
        # Run & check
        for bbcodes_text, expected_html_text in self.CUSTOM_TAGS_RENDERING_TESTS['tests']:
            result = self.parser.render(bbcodes_text)
            self.assertEqual(result, expected_html_text)

    def test_unicode_inputs(self):
        # Setup
        if sys.version_info >= (3,):
            src = '[center]ƒünk¥ • 你好 §tüƒƒ[/center]'
            dst = '
ƒünk¥ • 你好 §tüƒƒ
'
        else:
            src = unicode('[center]ƒünk¥ 你好 • §tüƒƒ 你好[/center]', 'utf-8')
            dst = unicode('
ƒünk¥ 你好 • §tüƒƒ 你好
', 'utf-8')
        # Run & check
        self.assertEqual(self.parser.render(src), dst)
<commit_msg>Placeholder regex tests started<commit_after># -*- coding: utf-8 -*-

# Standard library imports
import re
import sys

# Third party imports
from django.test import TestCase

# Local application / specific library imports
from precise_bbcode.parser import BBCodeParser
from precise_bbcode.parser import _text_re
from precise_bbcode.parser import _url_re

class ParserTestCase(TestCase):
    DEFAULT_TAGS_RENDERING_TESTS = (
        # BBcodes without errors
        ('[b]hello world![/b]', 'hello world!'),
        ('[b]hello [i]world![/i][/b]', 'hello world!'),
        ('[b]hello [ world![/b]', 'hello [ world!'),
        ('[b]]he[llo [ w]orld![/b]', ']he[llo [ w]orld!'),
        ('[ b ]hello [u]world![/u][ /b ]', 'hello world!'),
        ('[b]hello [] world![/b]', 'hello [] world!'),
        ('[list]\n[*]one\n[*]two\n[/list]', '

one

two

'),
        ('[list=1]\n[*]item 1\n[*]item 2\n[/list]', '

item 1

item 2

'),
        ('[list] [*]Item 1 [*]Item 2 [*]Item 3   [/list]', '

Item 1

Item 2

Item 3

'),
        ('>> some special chars >< <>', '>> some special chars >< <>'),
        ('"quoted text"', '"quoted text"'),
        ('>> some other special chars', '>> some other special chars'),
        ('[url]http://foo.com/bar.php?some--data[/url]', 'http://foo.com/bar.php?some--data'),
        ('[url]http://www.google.com[/url]', 'http://www.google.com'),
        ('[url=google.com]goto google[/url]', 'goto google'),
        ('[url=http://google.com][/url]', 'http://google.com'),
        ('[URL=google.com]goto google[/URL]', 'goto google'),
        ('[url=]xss[/url]', 'xss'),
        ('www.google.com foo.com/bar http://xyz.ci', 'www.google.com foo.com/bar http://xyz.ci'),
        ('[url=relative/foo/bar.html]link[/url]', 'link'),
        ('[url=/absolute/foo/bar.html]link[/url]', 'link'),
        ('[url=./hello.html]world![/url]', 'world!'),
        ('[img]http://www.foo.com/bar/img.png[/img]', ''),
        ('[quote] \r\nhello\nworld! [/quote]', '
hello
world!
'),
        ('[code][b]hello world![/b][/code]', '[b]hello world![/b]'),
        ('[color=green]goto [url=google.com]google website[/url][/color]', 'goto google website'),
        ('[color=#FFFFFF]white[/color]', 'white'),
        ('[color=]xss[/color]', '[color=<script></script>]xss[/color]'),
        ('[COLOR=blue]hello world![/color]', 'hello world!'),
        # BBCodes with syntactic errors
        ('[b]z sdf s s', '[b]z sdf s s'),
        ('[b][i]hello world![/b][/i]', '[i]hello world![/i]'),
        ('[b]hello [i]world![/i]', '[b]hello world!'),
        ('[color]test[/color]', '[color]test[/color]'),
        ('[/abcdef][/i]', '[/abcdef][/i]'),
        ('[b\n hello [i]the[/i] world![/b]', '[b
 hello the world![/b]'),
        ('[b]hello [i]the[/b] world![/i]', 'hello [i]the world![/i]'),
        ('[b] hello the[u]world ![/i] see you[/b]', ' hello the[u]world ![/i] see you'),
        # BBCodes with semantic errors
        ('[color=some words]test[/color]', '[color=some words]test[/color]'),
        # Unknown BBCodes
        ('[unknown][hello][/unknown]', '[unknown][hello][/unknown]'),
    )

    CUSTOM_TAGS_RENDERING_TESTS = {
        'tags': {
            'justify': {
                'args': ('justify', '[justify]{TEXT}[/justify]', '
{TEXT}
'),
                'kwargs': {},
            },
            'spoiler': {
                'args': ('spoiler', '[spoiler]{TEXT}[/spoiler]', '

    <input type="button" value="Afficher" style="width:60px;font-size:10px;margin:0px;padding:0px;" onclick="if (this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display != '') { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = '';        this.innerText = ''; this.value = \'Masquer\'; } else { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = \'none\'; this.innerText = ''; this.value = \'Afficher\'; }" />

{TEXT}

'),
                'kwargs': {},
            },
            'youtube': {
                'args': ('youtube', '[youtube]{TEXT}[/youtube]', '<object width="425" height="350"><param name="movie" value="http://www.youtube.com/v/{TEXT}"></param><param name="wmode" value="transparent"></param><embed src="http://www.youtube.com/v/{TEXT}" type="application/x-shockwave-flash" wmode="transparent" width="425" height="350"></embed></object>'),
                'kwargs': {},
            },
            'h1': {
                'args': ('h1', '[h1={COLOR}]{TEXT}[/h1]', '{TEXT}
'),
                'kwargs': {},
            },
            'hr': {
                'args': ('hr', '[hr]', '
'),
                'kwargs': {'standalone': True},
            },
            'size': {
                'args': ('size', '[size={NUMBER}]{TEXT}[/size]', '{TEXT}'),
                'kwargs': {},
            },
            'mailto': {
                'args': ('email', '[email]{EMAIL}[/email]', '{EMAIL}'),
                'kwargs': {'replace_links': False},
            },
            'simpletext': {
                'args': ('simpletext', '[simpletext]{SIMPLETEXT}[/simpletext]', '{SIMPLETEXT}'),
                'kwargs': {},
            }
        },
        'tests': (
            # BBcodes without errors
            ('[justify]hello world![/justify]', '
hello world!
'),
            ('[spoiler]hidden![/spoiler]', '

    <input type="button" value="Afficher" style="width:60px;font-size:10px;margin:0px;padding:0px;" onclick="if (this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display != '') { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = '';        this.innerText = ''; this.value = \'Masquer\'; } else { this.parentNode.parentNode.getElementsByTagName(\'div\')[1].getElementsByTagName(\'div\')[0].style.display = \'none\'; this.innerText = ''; this.value = \'Afficher\'; }" />

hidden!

'),
            ('[youtube]ztD3mRMdqSw[/youtube]', '<object width="425" height="350"><param name="movie" value="http://www.youtube.com/v/ztD3mRMdqSw"></param><param name="wmode" value="transparent"></param><embed src="http://www.youtube.com/v/ztD3mRMdqSw" type="application/x-shockwave-flash" wmode="transparent" width="425" height="350"></embed></object>'),
            ('[h1=#FFF]hello world![/h1]', 'hello world!
'),
            ('[hr]', '
'),
            ('[size=24]hello world![/size]', 'hello world!'),
            ('[email]xyz@xyz.com[/email]', 'xyz@xyz.com'),
            ('[email]xyz.fr@xyz.com[/email]', 'xyz.fr@xyz.com'),
            ('[simpletext]hello world[/simpletext]', 'hello world'),
            # BBCodes with semantic errors
            ('[size=]hello world![/size]', '[size]hello world![/size]'),
            ('[size=hello]hello world![/size]', '[size=hello]hello world![/size]'),
            ('[email]hello world![/email]', '[email]hello world![/email]'),
            ('[email]http://www.google.com[/email]', '[email]http://www.google.com[/email]'),
            ('[email=12]24[/email]', '[email]24[/email]'),
            ('[simpletext]hello world![/simpletext]', '[simpletext]hello world![/simpletext]'),
            ('[simpletext]hello #~ world[/simpletext]', '[simpletext]hello #~ world[/simpletext]'),
        )
    }

    PLACEHOLDERS_RE_TESTS = {
        'text': {
            're': _text_re,
            'tests': (
                u'hello world',
                u'hello\nworld',
                u'   hello world     ',
                u'http://asdf.xxxx.yyyy.com/vvvvv/PublicPages/Login.aspx?ReturnUrl=%2fvvvvv%2f(asdf@qwertybean.com/qwertybean)',
                u'12902',
            )
        },
        'url': {
            're': _url_re,
            'tests': (
                'http://foo.com/blah_blah',
                '(Something like http://foo.com/blah_blah)',
                'http://foo.com/blah_blah_(wikipedia)',
                'http://foo.com/more_(than)_one_(parens)',
                '(Something like http://foo.com/blah_blah_(wikipedia))',
                'http://foo.com/blah_(wikipedia)#cite-1',
                'http://foo.com/blah_(wikipedia)_blah#cite-1',
                'http://foo.com/(something)?after=parens',
                'http://foo.com/blah_blah.',
                'http://foo.com/blah_blah/.',
                '<http://foo.com/blah_blah>',
                '<http://foo.com/blah_blah/>',
                'http://foo.com/blah_blah,',
                'http://www.extinguishedscholar.com/wpglob/?p=364.',
                '<tag>http://example.com</tag>',
                'Just a www.example.com link.',
                'http://example.com/something?with,commas,in,url, but not at end',
                'bit.ly/foo',
                'http://asdf.xxxx.yyyy.com/vvvvv/PublicPages/Login.aspx?ReturnUrl=%2fvvvvv%2f(asdf@qwertybean.com/qwertybean)',
            )
        },
    }

    def setUp(self):
        self.parser = BBCodeParser()

    def test_default_tags_rendering(self):
        # Run & check
        for bbcodes_text, expected_html_text in self.DEFAULT_TAGS_RENDERING_TESTS:
            result = self.parser.render(bbcodes_text)
            self.assertEqual(result, expected_html_text)

    def test_custom_tags_rendering(self):
        # Setup
        for _, tag_def in self.CUSTOM_TAGS_RENDERING_TESTS['tags'].items():
            self.parser.add_default_renderer(*tag_def['args'], **tag_def['kwargs'])
        # Run & check
        for bbcodes_text, expected_html_text in self.CUSTOM_TAGS_RENDERING_TESTS['tests']:
            result = self.parser.render(bbcodes_text)
            self.assertEqual(result, expected_html_text)

    def test_unicode_inputs(self):
        # Setup
        if sys.version_info >= (3,):
            src = '[center]ƒünk¥ • 你好 §tüƒƒ[/center]'
            dst = '
ƒünk¥ • 你好 §tüƒƒ
'
        else:
            src = unicode('[center]ƒünk¥ 你好 • §tüƒƒ 你好[/center]', 'utf-8')
            dst = unicode('
ƒünk¥ 你好 • §tüƒƒ 你好
', 'utf-8')
        # Run & check
        self.assertEqual(self.parser.render(src), dst)

    def test_placeholder_regex(self):
        # Run & check
        for _, re_tests in self.PLACEHOLDERS_RE_TESTS.items():
            for test in re_tests['tests']:
                self.assertIsNotNone(re.search(re_tests['re'], test))
<|endoftext|>