<commit_before>from __future__ import absolute_import
import shutil
import os
from subprocess import check_output
from git.repo import Repo
from git.objects import Blob
from itertools import (count, izip, chain)
from kenja.git.util import (
                            commit_from_binsha,
                            mktree_from_iter,
                            write_tree
                    )
from multiprocessing import (
                                Pool,
                                cpu_count
                            )
from kenja.git.submodule import (
                                store_submodule_config,
                                get_submodule_tree_content
                    )

class SyntaxTreesCommitterBase:
    def __init__(self, org_repo, syntax_trees_dir):
        self.org_repo = org_repo
        self.syntax_trees_dir = syntax_trees_dir

    def is_completed_parse(self, blob):
        path = os.path.join(self.syntax_trees_dir, blob.hexsha)
        cmd = ['find', path, '-type', 'f']
        output = check_output(cmd)
        if len(output) == 0:
            #print 'Interface?:', blob.path
            pass
        return len(output) > 0

    def get_normalized_path(self, path):
        return path.replace("/", "_")

    def commit_syntax_trees(self, repo, changed_commits):
        start_commit = self.org_repo.commit(changed_commits.pop(0))
        total_commits = len(changed_commits)
        print '[00/%d] first commit to: %s' % (total_commits, repo.git_dir)
        self.construct_from_commit(repo, start_commit)

        for (num, commit_hexsha) in izip(count(1), changed_commits):
            print '[%d/%d] commit to: %s' % (num, total_commits, repo.git_dir)
            commit = self.org_repo.commit(commit_hexsha)
            self.apply_change(repo, commit)

    def is_commit_target(self, blob):
        if not blob.name.endswith('.java'):
            return False
        return self.is_completed_parse(blob)

class SyntaxTreesCommitter(SyntaxTreesCommitterBase):
    def __init__(self, org_repo, syntax_trees_dir):
        SyntaxTreesCommitterBase.__init__(self, org_repo, syntax_trees_dir)

    def remove_files(self, repo, index, removed_files):
        kwargs = {"r" : True}
        if len(removed_files) == 0:
            return
        index.remove(removed_files, **kwargs)
        index.write()

        for p in removed_files:
            shutil.rmtree(os.path.join(repo.working_dir, p))

    def add_files(self, repo, index, added_files):
        if len(added_files) == 0:
            return

        for path, hexsha in added_files.items():
            src = os.path.join(self.syntax_trees_dir, hexsha)
            dst = os.path.join(repo.working_dir, path)
            shutil.copytree(src, dst)

        repo.git.add(added_files.keys())
        index.update()

    def construct_from_commit(self, repo, commit):
        added_files = {}
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not self.is_commit_target(entry):
                continue
            added_files[self.get_normalized_path(entry.path)] = entry.hexsha

        self.add_files(repo, repo.index, added_files)
        repo.index.commit(commit.hexsha)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        index = new_repo.index
        removed_files = []
        added_files = {}
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not self.is_commit_target(diff.a_blob):
                        continue
                    removed_files.append(self.get_normalized_path(diff.a_blob.path))

                if(diff.b_blob):
                    if not self.is_commit_target(diff.b_blob):
                        continue
                    added_files[self.get_normalized_path(diff.b_blob.path)] = diff.b_blob.hexsha

            self.remove_files(new_repo, index, removed_files)
            self.add_files(new_repo, index, added_files)

        if len(index.diff(None, staged=True)):
            index.commit(commit.hexsha)

class FastSyntaxTreesCommitter(SyntaxTreesCommitterBase):
    def __init__(self, org_repo, syntax_trees_dir):
        SyntaxTreesCommitterBase.__init__(self, org_repo, syntax_trees_dir)
        self.previous_top_tree = {}

    def add_changed_blob(self, new_repo, blob):
        (mode, binsha) = self.write_syntax_tree(new_repo, blob)
        path = self.get_normalized_path(blob.path)
        self.previous_top_tree[path] = (mode, binsha)

    def commit(self, new_repo, org_commit):
        submodule_info = []
        submodule_info.append((self.submodule_conf_mode, self.submodule_conf_binsha, '.gitmodules'))
        submodule_info.append(get_submodule_tree_content(org_commit.hexsha, 'org_repo'))
        syntax_trees_tree_contents = self.iter_object_info()
        (mode, binsha) = mktree_from_iter(new_repo.odb, chain(syntax_trees_tree_contents, submodule_info))

        message = org_commit.message.encode(org_commit.encoding)
        commit_from_binsha(new_repo, binsha, message)

    def construct_from_commit(self, repo, commit):
        mode, binsha = store_submodule_config(repo.odb, 'original', 'org_repo', self.org_repo.git_dir)
        self.submodule_conf_binsha = binsha
        self.submodule_conf_mode = mode
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not self.is_commit_target(entry):
                continue
            self.add_changed_blob(repo, entry)

        self.commit(repo, commit)

    def write_syntax_tree(self, repo, blob):
        src = os.path.join(self.syntax_trees_dir, blob.hexsha)
        return write_tree(repo.odb, src)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        changed = False
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not self.is_commit_target(diff.a_blob):
                        continue
                    path = self.get_normalized_path(diff.a_blob.path)
                    self.previous_top_tree.pop(path)
                    changed = True

                if(diff.b_blob):
                    if not self.is_commit_target(diff.b_blob):
                        continue
                    self.add_changed_blob(new_repo, diff.b_blob)
                    changed = True

        if changed:
            self.commit(new_repo, commit)

    def iter_object_info(self):
        for (name, (mode, binsha)) in self.previous_top_tree.items():
            yield mode, binsha, name

def commit_syntax_trees_worker(repo_dir, org_repo_dir, changed_commits, syntax_trees_dir, syntax_trees_committer):
    repo = Repo(repo_dir)
    org_repo = Repo(org_repo_dir)
    committer = syntax_trees_committer(org_repo, syntax_trees_dir)
    committer.commit_syntax_trees(repo, changed_commits)

class SyntaxTreesParallelCommitter:
    def __init__(self, syntax_trees_dir, org_repo_dir, syntax_trees_committer, processes=None):
        self.syntax_trees_dir = syntax_trees_dir
        self.org_repo_dir = org_repo_dir
        self.processes = processes if processes else cpu_count()
        self.pool = Pool(self.processes)
        self.closed = False
        self.syntax_trees_committer = syntax_trees_committer

    def commit_syntax_trees_parallel(self, repo_dir, changed_commits):
        args = [repo_dir, self.org_repo_dir, changed_commits, self.syntax_trees_dir, self.syntax_trees_committer]
        if(self.closed):
            self.pool = Pool(self.processes)
            self.closed = False

        self.pool.apply_async(commit_syntax_trees_worker, args=args)

    def join(self):
        self.pool.close()
        self.closed = True
        self.pool.join()
        self.pool = None
<commit_msg>メソッドの位置を変更した．<commit_after>from __future__ import absolute_import
import shutil
import os
from subprocess import check_output
from git.repo import Repo
from git.objects import Blob
from itertools import (count, izip, chain)
from kenja.git.util import (
                            commit_from_binsha,
                            mktree_from_iter,
                            write_tree
                    )
from multiprocessing import (
                                Pool,
                                cpu_count
                            )
from kenja.git.submodule import (
                                store_submodule_config,
                                get_submodule_tree_content
                    )

class SyntaxTreesCommitterBase:
    def __init__(self, org_repo, syntax_trees_dir):
        self.org_repo = org_repo
        self.syntax_trees_dir = syntax_trees_dir

    def is_completed_parse(self, blob):
        path = os.path.join(self.syntax_trees_dir, blob.hexsha)
        cmd = ['find', path, '-type', 'f']
        output = check_output(cmd)
        if len(output) == 0:
            #print 'Interface?:', blob.path
            pass
        return len(output) > 0

    def is_commit_target(self, blob):
        if not blob.name.endswith('.java'):
            return False
        return self.is_completed_parse(blob)

    def get_normalized_path(self, path):
        return path.replace("/", "_")

    def commit_syntax_trees(self, repo, changed_commits):
        start_commit = self.org_repo.commit(changed_commits.pop(0))
        total_commits = len(changed_commits)
        print '[00/%d] first commit to: %s' % (total_commits, repo.git_dir)
        self.construct_from_commit(repo, start_commit)

        for (num, commit_hexsha) in izip(count(1), changed_commits):
            print '[%d/%d] commit to: %s' % (num, total_commits, repo.git_dir)
            commit = self.org_repo.commit(commit_hexsha)
            self.apply_change(repo, commit)

class SyntaxTreesCommitter(SyntaxTreesCommitterBase):
    def __init__(self, org_repo, syntax_trees_dir):
        SyntaxTreesCommitterBase.__init__(self, org_repo, syntax_trees_dir)

    def remove_files(self, repo, index, removed_files):
        kwargs = {"r" : True}
        if len(removed_files) == 0:
            return
        index.remove(removed_files, **kwargs)
        index.write()

        for p in removed_files:
            shutil.rmtree(os.path.join(repo.working_dir, p))

    def add_files(self, repo, index, added_files):
        if len(added_files) == 0:
            return

        for path, hexsha in added_files.items():
            src = os.path.join(self.syntax_trees_dir, hexsha)
            dst = os.path.join(repo.working_dir, path)
            shutil.copytree(src, dst)

        repo.git.add(added_files.keys())
        index.update()

    def construct_from_commit(self, repo, commit):
        added_files = {}
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not self.is_commit_target(entry):
                continue
            added_files[self.get_normalized_path(entry.path)] = entry.hexsha

        self.add_files(repo, repo.index, added_files)
        repo.index.commit(commit.hexsha)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        index = new_repo.index
        removed_files = []
        added_files = {}
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not self.is_commit_target(diff.a_blob):
                        continue
                    removed_files.append(self.get_normalized_path(diff.a_blob.path))

                if(diff.b_blob):
                    if not self.is_commit_target(diff.b_blob):
                        continue
                    added_files[self.get_normalized_path(diff.b_blob.path)] = diff.b_blob.hexsha

            self.remove_files(new_repo, index, removed_files)
            self.add_files(new_repo, index, added_files)

        if len(index.diff(None, staged=True)):
            index.commit(commit.hexsha)

class FastSyntaxTreesCommitter(SyntaxTreesCommitterBase):
    def __init__(self, org_repo, syntax_trees_dir):
        SyntaxTreesCommitterBase.__init__(self, org_repo, syntax_trees_dir)
        self.previous_top_tree = {}

    def add_changed_blob(self, new_repo, blob):
        (mode, binsha) = self.write_syntax_tree(new_repo, blob)
        path = self.get_normalized_path(blob.path)
        self.previous_top_tree[path] = (mode, binsha)

    def commit(self, new_repo, org_commit):
        submodule_info = []
        submodule_info.append((self.submodule_conf_mode, self.submodule_conf_binsha, '.gitmodules'))
        submodule_info.append(get_submodule_tree_content(org_commit.hexsha, 'org_repo'))
        syntax_trees_tree_contents = self.iter_object_info()
        (mode, binsha) = mktree_from_iter(new_repo.odb, chain(syntax_trees_tree_contents, submodule_info))

        message = org_commit.message.encode(org_commit.encoding)
        commit_from_binsha(new_repo, binsha, message)

    def construct_from_commit(self, repo, commit):
        mode, binsha = store_submodule_config(repo.odb, 'original', 'org_repo', self.org_repo.git_dir)
        self.submodule_conf_binsha = binsha
        self.submodule_conf_mode = mode
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not self.is_commit_target(entry):
                continue
            self.add_changed_blob(repo, entry)

        self.commit(repo, commit)

    def write_syntax_tree(self, repo, blob):
        src = os.path.join(self.syntax_trees_dir, blob.hexsha)
        return write_tree(repo.odb, src)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        changed = False
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not self.is_commit_target(diff.a_blob):
                        continue
                    path = self.get_normalized_path(diff.a_blob.path)
                    self.previous_top_tree.pop(path)
                    changed = True

                if(diff.b_blob):
                    if not self.is_commit_target(diff.b_blob):
                        continue
                    self.add_changed_blob(new_repo, diff.b_blob)
                    changed = True

        if changed:
            self.commit(new_repo, commit)

    def iter_object_info(self):
        for (name, (mode, binsha)) in self.previous_top_tree.items():
            yield mode, binsha, name

def commit_syntax_trees_worker(repo_dir, org_repo_dir, changed_commits, syntax_trees_dir, syntax_trees_committer):
    repo = Repo(repo_dir)
    org_repo = Repo(org_repo_dir)
    committer = syntax_trees_committer(org_repo, syntax_trees_dir)
    committer.commit_syntax_trees(repo, changed_commits)

class SyntaxTreesParallelCommitter:
    def __init__(self, syntax_trees_dir, org_repo_dir, syntax_trees_committer, processes=None):
        self.syntax_trees_dir = syntax_trees_dir
        self.org_repo_dir = org_repo_dir
        self.processes = processes if processes else cpu_count()
        self.pool = Pool(self.processes)
        self.closed = False
        self.syntax_trees_committer = syntax_trees_committer

    def commit_syntax_trees_parallel(self, repo_dir, changed_commits):
        args = [repo_dir, self.org_repo_dir, changed_commits, self.syntax_trees_dir, self.syntax_trees_committer]
        if(self.closed):
            self.pool = Pool(self.processes)
            self.closed = False

        self.pool.apply_async(commit_syntax_trees_worker, args=args)

    def join(self):
        self.pool.close()
        self.closed = True
        self.pool.join()
        self.pool = None
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
from tempfile import mkdtemp
from shutil import rmtree
from itertools import count, izip
from git.repo import Repo
from git.objects import Blob
from kenja.parser import JavaParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesCommitter
from logging import getLogger

logger = getLogger(__name__)

class HistorageConverter:
    def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        self.language = '.java'
        self.check_and_make_working_dir(historage_dir)
        self.historage_dir = historage_dir

        self.use_tempdir = syntax_trees_dir is None
        if self.use_tempdir:
            self.syntax_trees_dir = mkdtemp()
            logger.info(self.syntax_trees_dir)
        else:
            self.check_and_make_working_dir(syntax_trees_dir)
            self.syntax_trees_dir = syntax_trees_dir

        self.num_commits = 0

        self.is_bare_repo = False

    def check_and_make_working_dir(self, path):
        if os.path.isdir(path):
            if os.listdir(path):
                raise Exception('{0} is not an empty directory'.format(path))
        else:
            try:
                os.mkdir(path)
            except OSError:
                logger.error('Kenja cannot make a directory: {0}'.format(path))
                raise

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_target_files(self):
        logger.info('create parser processes...')
        parser_executor = JavaParserExecutor(self.syntax_trees_dir)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            commit = self.org_repo.commit(commit)
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, self.language):
                            if diff.b_blob.hexsha not in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, self.language):
                        if entry.hexsha not in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        logger.info('waiting parser processes')
        parser_executor.join()

    def prepare_historage_repo(self):
        historage_repo = Repo.init(self.historage_dir, bare=self.is_bare_repo)
        self.set_git_config(historage_repo)
        return historage_repo

    def set_git_config(self, repo):
        reader = repo.config_reader()  # global config
        writer = repo.config_writer()  # local config
        user_key = 'user'
        if not reader.has_option(user_key, 'name'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'name', 'Kenja Converter')
        if not reader.has_option(user_key, 'email'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'email', 'default@example.com')

    def convert(self):
        self.parse_all_target_files()
        self.construct_historage()

    def construct_historage(self):
        logger.info('convert a git repository to a  historage...')

        historage_repo = self.prepare_historage_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), historage_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            commit = self.org_repo.commit(commit)
            logger.info('[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, historage_repo.git_dir))
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()
        if not self.is_bare_repo:
            historage_repo.head.reset(working_tree=True)
        logger.info('completed!')

    def __del__(self):
        if self.use_tempdir and os.path.exists(self.syntax_trees_dir):
            rmtree(self.syntax_trees_dir)
<commit_msg>get_reversed_topological_ordered_commits returns commits<commit_after>from __future__ import absolute_import
import os
from tempfile import mkdtemp
from shutil import rmtree
from itertools import count, izip
from git.repo import Repo
from git.objects import Blob
from kenja.parser import JavaParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesCommitter
from logging import getLogger

logger = getLogger(__name__)

class HistorageConverter:
    def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        self.language = '.java'
        self.check_and_make_working_dir(historage_dir)
        self.historage_dir = historage_dir

        self.use_tempdir = syntax_trees_dir is None
        if self.use_tempdir:
            self.syntax_trees_dir = mkdtemp()
            logger.info(self.syntax_trees_dir)
        else:
            self.check_and_make_working_dir(syntax_trees_dir)
            self.syntax_trees_dir = syntax_trees_dir

        self.num_commits = 0

        self.is_bare_repo = False

    def check_and_make_working_dir(self, path):
        if os.path.isdir(path):
            if os.listdir(path):
                raise Exception('{0} is not an empty directory'.format(path))
        else:
            try:
                os.mkdir(path)
            except OSError:
                logger.error('Kenja cannot make a directory: {0}'.format(path))
                raise

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_target_files(self):
        logger.info('create parser processes...')
        parser_executor = JavaParserExecutor(self.syntax_trees_dir)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, self.language):
                            if diff.b_blob.hexsha not in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, self.language):
                        if entry.hexsha not in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        logger.info('waiting parser processes')
        parser_executor.join()

    def prepare_historage_repo(self):
        historage_repo = Repo.init(self.historage_dir, bare=self.is_bare_repo)
        self.set_git_config(historage_repo)
        return historage_repo

    def set_git_config(self, repo):
        reader = repo.config_reader()  # global config
        writer = repo.config_writer()  # local config
        user_key = 'user'
        if not reader.has_option(user_key, 'name'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'name', 'Kenja Converter')
        if not reader.has_option(user_key, 'email'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'email', 'default@example.com')

    def convert(self):
        self.parse_all_target_files()
        self.construct_historage()

    def construct_historage(self):
        logger.info('convert a git repository to a  historage...')

        historage_repo = self.prepare_historage_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), historage_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            logger.info('[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, historage_repo.git_dir))
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()
        if not self.is_bare_repo:
            historage_repo.head.reset(working_tree=True)
        logger.info('completed!')

    def __del__(self):
        if self.use_tempdir and os.path.exists(self.syntax_trees_dir):
            rmtree(self.syntax_trees_dir)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from numbers import Number
try:
    import xlsxwriter
    import xlrd
except ImportError:
    xlsxwriter = None
    xlrd = None

try:
    basestring
except NameError:
    basestring = str

def get_sheet_from_file(directory, sheet_idx=None, sheet_name=None):
    """both idx and name could be a list, then a list of sheets will be returned."""
    if sheet_idx is not None:
        if isinstance(sheet_idx, int):
            return xlrd.open_workbook(directory).sheet_by_index(sheet_idx)
        elif isinstance(sheet_idx, list):
            return [xlrd.open_workbook(directory).sheet_by_index(idx) for idx in sheet_idx]
        else:
            raise TypeError('invalid sheet_idx: %s' % sheet_idx)
    elif sheet_name is not None:
        if isinstance(sheet_idx, basestring):
            return xlrd.open_workbook(directory).sheet_by_name(sheet_name)
        elif isinstance(sheet_idx, list):
            return [xlrd.open_workbook(directory).sheet_by_name(name) for name in sheet_name]
        else:
            raise TypeError('invalid sheet_name: %s' % sheet_name)
    else:
        raise ValueError('both sheet_idx & sheet_name is None, what do you want?')

def get_subframe(sheet, col_idxs, row_offset, row_limit=None):
    """get sub dataframe from sheet
    return value is like:
    [[1, 2, 3],
     [8, 9, 0]]
    """
    df = []
    row_range = row_offset + row_limit if row_limit else sheet.nrows
    for i in range(row_offset, row_range):
        df.append([sheet.cell(i, j).value for j in col_idxs])
    return df

def write_excel(file_name, sheets=[]):
    """sheet is like:
    {
        'name': 'sheet1',
        'data': [[1, 2, 3],
                 [8, 9, 0]]
    }
    """
    workbook = xlsxwriter.Workbook('%s.xlsx' % file_name)
    for sheet in sheets:
        worksheet = workbook.add_worksheet(name=sheet['name'])
        for i in range(len(sheet['data'])):
            for j in range(len(sheet['data'][i])):
                if isinstance(sheet['data'][i][j], Number):
                    worksheet.write_number(i, j, int(sheet['data'][i][j]))
                elif isinstance(sheet['data'][i][j], basestring):
                    worksheet.write_string(i, j, sheet['data'][i][j])
                else:
                    worksheet.write_blank(i, j, sheet['data'][i][j])
    workbook.close()
<commit_msg>update write excel<commit_after># -*- coding: utf-8 -*-
from numbers import Number
try:
    import xlsxwriter
    import xlrd
except ImportError:
    xlsxwriter = None
    xlrd = None

try:
    basestring
except NameError:
    basestring = str

def get_sheet_from_file(directory, sheet_idx=None, sheet_name=None):
    """both idx and name could be a list, then a list of sheets will be returned."""
    if sheet_idx is not None:
        if isinstance(sheet_idx, int):
            return xlrd.open_workbook(directory).sheet_by_index(sheet_idx)
        elif isinstance(sheet_idx, list):
            return [xlrd.open_workbook(directory).sheet_by_index(idx) for idx in sheet_idx]
        else:
            raise TypeError('invalid sheet_idx: %s' % sheet_idx)
    elif sheet_name is not None:
        if isinstance(sheet_idx, basestring):
            return xlrd.open_workbook(directory).sheet_by_name(sheet_name)
        elif isinstance(sheet_idx, list):
            return [xlrd.open_workbook(directory).sheet_by_name(name) for name in sheet_name]
        else:
            raise TypeError('invalid sheet_name: %s' % sheet_name)
    else:
        raise ValueError('both sheet_idx & sheet_name is None, what do you want?')

def get_subframe(sheet, col_idxs, row_offset, row_limit=None):
    """get sub dataframe from sheet
    return value is like:
    [[1, 2, 3],
     [8, 9, 0]]
    """
    df = []
    row_range = row_offset + row_limit if row_limit else sheet.nrows
    for i in range(row_offset, row_range):
        df.append([sheet.cell(i, j).value for j in col_idxs])
    return df

def write_excel(file_path, sheets={}):
    """sheets is like:
    {
        'sheet1':[[1, 2, 3],
                  [8, 9, 0]],
        'sheet2':[[]],...
    }
    """
    workbook = xlsxwriter.Workbook(file_path)
    for name, data in sheets.items():
        worksheet = workbook.add_worksheet(name=name)
        for i in range(len(data)):
            for j in range(len(data[i])):
                if isinstance(data[i][j], Number):
                    worksheet.write_number(i, j, int(data[i][j]))
                elif isinstance(data[i][j], basestring):
                    worksheet.write_string(i, j, data[i][j])
                else:
                    worksheet.write_blank(i, j, data[i][j])
    workbook.close()
<|endoftext|>


<commit_before>PLUGIN_VERSION = '2.8.5'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<commit_msg>Tag new release: 2.8.6<commit_after>PLUGIN_VERSION = '2.8.6'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Dump the state of the world for post mortem."""

import argparse
import datetime
from distutils import spawn
import fnmatch
import os
import os.path
import subprocess
import sys

GMR_PROCESSES = (
    'nova-compute',
    'neutron-dhcp-agent',
    'neutron-l3-agent',
    'neutron-linuxbridge-agent',
    'neutron-metadata-agent',
    'neutron-openvswitch-agent',
    'cinder-volume',
)

def get_options():
    parser = argparse.ArgumentParser(
        description='Dump world state for debugging')
    parser.add_argument('-d', '--dir',
                        default='.',
                        help='Output directory for worlddump')
    parser.add_argument('-n', '--name',
                        default='',
                        help='Additional name to tag into file')
    return parser.parse_args()

def filename(dirname, name=""):
    now = datetime.datetime.utcnow()
    fmt = "worlddump-%Y-%m-%d-%H%M%S"
    if name:
        fmt += "-" + name
    fmt += ".txt"
    return os.path.join(dirname, now.strftime(fmt))

def warn(msg):
    print("WARN: %s" % msg)

def _dump_cmd(cmd):
    print(cmd)
    print("-" * len(cmd))
    print()
    try:
        subprocess.check_call(cmd, shell=True)
        print()
    except subprocess.CalledProcessError as e:
        print("*** Failed to run '%(cmd)s': %(err)s" % {'cmd': cmd, 'err': e})

def _find_cmd(cmd):
    if not spawn.find_executable(cmd):
        print("*** %s not found: skipping" % cmd)
        return False
    return True

def _header(name):
    print()
    print(name)
    print("=" * len(name))
    print()

def _bridge_list():
    process = subprocess.Popen(['sudo', 'ovs-vsctl', 'list-br'],
                               stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    return stdout.split()

# This method gets a max openflow version supported by openvswitch.
# For example 'ovs-ofctl --version' displays the following:
#
#     ovs-ofctl (Open vSwitch) 2.0.2
#     Compiled Dec  9 2015 14:08:08
#     OpenFlow versions 0x1:0x4
#
# The above shows that openvswitch supports from OpenFlow10 to OpenFlow13.
# This method gets max version searching 'OpenFlow versions 0x1:0x'.
# And return a version value converted to an integer type.
def _get_ofp_version():
    process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    find_str = 'OpenFlow versions 0x1:0x'
    offset = stdout.find(find_str)
    return int(stdout[offset + len(find_str):-1]) - 1

def disk_space():
    # the df output
    _header("File System Summary")

    dfraw = os.popen("df -Ph").read()
    df = [s.split() for s in dfraw.splitlines()]
    for fs in df:
        try:
            if int(fs[4][:-1]) > 95:
                warn("Device %s (%s) is %s full, might be an issue" % (
                    fs[0], fs[5], fs[4]))
        except ValueError:
            # if it doesn't look like an int, that's fine
            pass

    print(dfraw)

def ebtables_dump():
    tables = ['filter', 'nat', 'broute']
    _header("EB Tables Dump")
    if not _find_cmd('ebtables'):
        return
    for table in tables:
        _dump_cmd("sudo ebtables -t %s -L" % table)

def iptables_dump():
    tables = ['filter', 'nat', 'mangle']
    _header("IP Tables Dump")

    for table in tables:
        _dump_cmd("sudo iptables --line-numbers -L -nv -t %s" % table)

def _netns_list():
    process = subprocess.Popen(['ip', 'netns'], stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    return stdout.split()

def network_dump():
    _header("Network Dump")

    _dump_cmd("brctl show")
    _dump_cmd("arp -n")
    ip_cmds = ["addr", "link", "route"]
    for cmd in ip_cmds + ['netns']:
        _dump_cmd("ip %s" % cmd)
    for netns_ in _netns_list():
        for cmd in ip_cmds:
            args = {'netns': netns_, 'cmd': cmd}
            _dump_cmd('sudo ip netns exec %(netns)s ip %(cmd)s' % args)

def ovs_dump():
    _header("Open vSwitch Dump")

    # NOTE(cdent): If we're not using neutron + ovs these commands
    # will not be present so
    if not _find_cmd('ovs-vsctl'):
        return

    bridges = _bridge_list()
    ofctl_cmds = ('show', 'dump-ports-desc', 'dump-ports', 'dump-flows')
    ofp_max = _get_ofp_version()
    vers = 'OpenFlow10'
    for i in range(1, ofp_max + 1):
        vers += ',OpenFlow1' + str(i)
    _dump_cmd("sudo ovs-vsctl show")
    for ofctl_cmd in ofctl_cmds:
        for bridge in bridges:
            args = {'vers': vers, 'cmd': ofctl_cmd, 'bridge': bridge}
            _dump_cmd("sudo ovs-ofctl --protocols=%(vers)s %(cmd)s %(bridge)s" % args)

def process_list():
    _header("Process Listing")
    _dump_cmd("ps axo "
              "user,ppid,pid,pcpu,pmem,vsz,rss,tty,stat,start,time,args")

def compute_consoles():
    _header("Compute consoles")
    for root, dirnames, filenames in os.walk('/opt/stack'):
        for filename in fnmatch.filter(filenames, 'console.log'):
            fullpath = os.path.join(root, filename)
            _dump_cmd("sudo cat %s" % fullpath)

def guru_meditation_reports():
    for service in GMR_PROCESSES:
        _header("%s Guru Meditation Report" % service)

        try:
            subprocess.check_call(['pgrep', '-f', service])
        except subprocess.CalledProcessError:
            print("Skipping as %s does not appear to be running" % service)
            continue

        _dump_cmd("killall -e -USR2 %s" % service)
        print("guru meditation report in %s log" % service)

def main():
    opts = get_options()
    fname = filename(opts.dir, opts.name)
    print("World dumping... see %s for details" % fname)
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    with open(fname, 'w') as f:
        os.dup2(f.fileno(), sys.stdout.fileno())
        disk_space()
        process_list()
        network_dump()
        ovs_dump()
        iptables_dump()
        ebtables_dump()
        compute_consoles()
        guru_meditation_reports()

if __name__ == '__main__':
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)
<commit_msg>Fix error in 'ip netns' parsing<commit_after>#!/usr/bin/env python
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Dump the state of the world for post mortem."""

import argparse
import datetime
from distutils import spawn
import fnmatch
import os
import os.path
import subprocess
import sys

GMR_PROCESSES = (
    'nova-compute',
    'neutron-dhcp-agent',
    'neutron-l3-agent',
    'neutron-linuxbridge-agent',
    'neutron-metadata-agent',
    'neutron-openvswitch-agent',
    'cinder-volume',
)

def get_options():
    parser = argparse.ArgumentParser(
        description='Dump world state for debugging')
    parser.add_argument('-d', '--dir',
                        default='.',
                        help='Output directory for worlddump')
    parser.add_argument('-n', '--name',
                        default='',
                        help='Additional name to tag into file')
    return parser.parse_args()

def filename(dirname, name=""):
    now = datetime.datetime.utcnow()
    fmt = "worlddump-%Y-%m-%d-%H%M%S"
    if name:
        fmt += "-" + name
    fmt += ".txt"
    return os.path.join(dirname, now.strftime(fmt))

def warn(msg):
    print("WARN: %s" % msg)

def _dump_cmd(cmd):
    print(cmd)
    print("-" * len(cmd))
    print()
    try:
        subprocess.check_call(cmd, shell=True)
        print()
    except subprocess.CalledProcessError as e:
        print("*** Failed to run '%(cmd)s': %(err)s" % {'cmd': cmd, 'err': e})

def _find_cmd(cmd):
    if not spawn.find_executable(cmd):
        print("*** %s not found: skipping" % cmd)
        return False
    return True

def _header(name):
    print()
    print(name)
    print("=" * len(name))
    print()

def _bridge_list():
    process = subprocess.Popen(['sudo', 'ovs-vsctl', 'list-br'],
                               stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    return stdout.split()

# This method gets a max openflow version supported by openvswitch.
# For example 'ovs-ofctl --version' displays the following:
#
#     ovs-ofctl (Open vSwitch) 2.0.2
#     Compiled Dec  9 2015 14:08:08
#     OpenFlow versions 0x1:0x4
#
# The above shows that openvswitch supports from OpenFlow10 to OpenFlow13.
# This method gets max version searching 'OpenFlow versions 0x1:0x'.
# And return a version value converted to an integer type.
def _get_ofp_version():
    process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    find_str = 'OpenFlow versions 0x1:0x'
    offset = stdout.find(find_str)
    return int(stdout[offset + len(find_str):-1]) - 1

def disk_space():
    # the df output
    _header("File System Summary")

    dfraw = os.popen("df -Ph").read()
    df = [s.split() for s in dfraw.splitlines()]
    for fs in df:
        try:
            if int(fs[4][:-1]) > 95:
                warn("Device %s (%s) is %s full, might be an issue" % (
                    fs[0], fs[5], fs[4]))
        except ValueError:
            # if it doesn't look like an int, that's fine
            pass

    print(dfraw)

def ebtables_dump():
    tables = ['filter', 'nat', 'broute']
    _header("EB Tables Dump")
    if not _find_cmd('ebtables'):
        return
    for table in tables:
        _dump_cmd("sudo ebtables -t %s -L" % table)

def iptables_dump():
    tables = ['filter', 'nat', 'mangle']
    _header("IP Tables Dump")

    for table in tables:
        _dump_cmd("sudo iptables --line-numbers -L -nv -t %s" % table)

def _netns_list():
    process = subprocess.Popen(['ip', 'netns'], stdout=subprocess.PIPE)
    stdout, _ = process.communicate()
    # NOTE(jlvillal): Sometimes 'ip netns list' can return output like:
    #   qrouter-0805fd7d-c493-4fa6-82ca-1c6c9b23cd9e (id: 1)
    #   qdhcp-bb2cc6ae-2ae8-474f-adda-a94059b872b5 (id: 0)
    output = [x.split()[0] for x in stdout.splitlines()]
    return output

def network_dump():
    _header("Network Dump")

    _dump_cmd("brctl show")
    _dump_cmd("arp -n")
    ip_cmds = ["addr", "link", "route"]
    for cmd in ip_cmds + ['netns']:
        _dump_cmd("ip %s" % cmd)
    for netns_ in _netns_list():
        for cmd in ip_cmds:
            args = {'netns': netns_, 'cmd': cmd}
            _dump_cmd('sudo ip netns exec %(netns)s ip %(cmd)s' % args)

def ovs_dump():
    _header("Open vSwitch Dump")

    # NOTE(cdent): If we're not using neutron + ovs these commands
    # will not be present so
    if not _find_cmd('ovs-vsctl'):
        return

    bridges = _bridge_list()
    ofctl_cmds = ('show', 'dump-ports-desc', 'dump-ports', 'dump-flows')
    ofp_max = _get_ofp_version()
    vers = 'OpenFlow10'
    for i in range(1, ofp_max + 1):
        vers += ',OpenFlow1' + str(i)
    _dump_cmd("sudo ovs-vsctl show")
    for ofctl_cmd in ofctl_cmds:
        for bridge in bridges:
            args = {'vers': vers, 'cmd': ofctl_cmd, 'bridge': bridge}
            _dump_cmd("sudo ovs-ofctl --protocols=%(vers)s %(cmd)s %(bridge)s" % args)

def process_list():
    _header("Process Listing")
    _dump_cmd("ps axo "
              "user,ppid,pid,pcpu,pmem,vsz,rss,tty,stat,start,time,args")

def compute_consoles():
    _header("Compute consoles")
    for root, dirnames, filenames in os.walk('/opt/stack'):
        for filename in fnmatch.filter(filenames, 'console.log'):
            fullpath = os.path.join(root, filename)
            _dump_cmd("sudo cat %s" % fullpath)

def guru_meditation_reports():
    for service in GMR_PROCESSES:
        _header("%s Guru Meditation Report" % service)

        try:
            subprocess.check_call(['pgrep', '-f', service])
        except subprocess.CalledProcessError:
            print("Skipping as %s does not appear to be running" % service)
            continue

        _dump_cmd("killall -e -USR2 %s" % service)
        print("guru meditation report in %s log" % service)

def main():
    opts = get_options()
    fname = filename(opts.dir, opts.name)
    print("World dumping... see %s for details" % fname)
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    with open(fname, 'w') as f:
        os.dup2(f.fileno(), sys.stdout.fileno())
        disk_space()
        process_list()
        network_dump()
        ovs_dump()
        iptables_dump()
        ebtables_dump()
        compute_consoles()
        guru_meditation_reports()

if __name__ == '__main__':
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)
<|endoftext|>


<commit_before>### CONSTANTS ###
FL_VERSION = "2.1.1"

######## HEATMAP #############
DEFAULT_BINSIZE = 100
DEFAULT_METRIC = "e"  # Euclidian, PyCluster

DEFAULT_SCALE = "90%"
DEFAULT_EXTEND = 5000
DEFAULT_PERCENTILE = 99
DEFAULT_CLUSTERING = "none"
DEFAULT_BG = "white"
DEFAULT_DYN_EXTEND = 5000

######## BANDPLOT #############
# Sizes of the plots (in inches)
PLOTWIDTH = 0.6
PLOTHEIGHT = 0.6
PAD = 0.05
PADLEFT = 1.5  # For labels
PADTOP = 0.4  # For labels
PADBOTTOM = 0.05
PADRIGHT = 0.05
BINS = 100  # Number of bins for profile
SCALAR = 99

######## PROFILE #############
BACKGROUNDS = ["white", "stripes", "color"]
FRAGMENTLENGTH = 200
PROFILE_MIN_Y = 75

######## OTHER #############
FONTSIZE = 8
<commit_msg>version bump<commit_after>### CONSTANTS ###
FL_VERSION = "2.1.2"

######## HEATMAP #############
DEFAULT_BINSIZE = 100
DEFAULT_METRIC = "e"  # Euclidian, PyCluster

DEFAULT_SCALE = "90%"
DEFAULT_EXTEND = 5000
DEFAULT_PERCENTILE = 99
DEFAULT_CLUSTERING = "none"
DEFAULT_BG = "white"
DEFAULT_DYN_EXTEND = 5000

######## BANDPLOT #############
# Sizes of the plots (in inches)
PLOTWIDTH = 0.6
PLOTHEIGHT = 0.6
PAD = 0.05
PADLEFT = 1.5  # For labels
PADTOP = 0.4  # For labels
PADBOTTOM = 0.05
PADRIGHT = 0.05
BINS = 100  # Number of bins for profile
SCALAR = 99

######## PROFILE #############
BACKGROUNDS = ["white", "stripes", "color"]
FRAGMENTLENGTH = 200
PROFILE_MIN_Y = 75

######## OTHER #############
FONTSIZE = 8
<|endoftext|>


<commit_before>#! /usr/bin/env python
#-----------------------------------------------------------------------
# COPYRIGHT_BEGIN
# Copyright (C) 2016-2017, FixFlyer, LLC.
# All rights reserved.
# COPYRIGHT_END
#-----------------------------------------------------------------------

"""Logging support."""

import os
import time
import psutil

# Emergency message level.
EMERG = 0

# Alert message level.
ALERT = 1

# Critical message level.
CRIT = 2

# Error message level.
ERR = 3

# Warning message level.
WARNING = 4

# Notice message level.
NOTICE = 5

# Informational message level.
INFO = 6

# Debugging message level.
DEBUG = 7

# Level names.
NAMES = ["EMERG", "ALERT", "CRIT", "ERR", "WARNING", "NOTICE", "INFO", "DEBUG"]

# Level numbers.
LEVELS = {"EMERG": EMERG,
          "ALERT": ALERT,
          "CRIT": CRIT,
          "ERR": ERR,
          "WARNING": WARNING,
          "NOTICE": NOTICE,
          "INFO": INFO,
          "DEBUG": DEBUG}

class Logger(object):
    """Logger interface.

    The API library logs informational messages describing its
    internal state and events.  By default, these messages are logged
    to a file in the host system's temporary directory.

    It's often desirable to redirect this logging to whatever facility
    is used by the client application.  The library's logging can be
    redirected by using an adaptor that implements this interface.
    """

    @staticmethod
    def string_to_level(level):
        """Convert a string level name to its integer value.

        @param[in] level
            A level name, matching the constants defined above (EMERG, etc).

        @retval @c None
            No level name matched @p level.

        @returns
            Integer level value."""

        return LEVELS.get(level)

    @staticmethod
    def level_to_string(level):
        """Convert an integer level into its string name.

        @param[in] level
            Integer level value.

        @retval None
            The supplied @p level value is invalid.

        @returns
            String name for the specified level."""

        if level < 0 or level >= len(NAMES):
            return None
        return NAMES[level]

    def create_log(self, log):
        """Create a new log instance.

        @param[in] log
            Name of log instance.

        @retval 0
           Successful."""
        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def set_log_level(self, name, level):
        """Set current minimum log level.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""

        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def get_log_level(self, name):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @param[in] name
            Name of a log instance.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""

        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def log(self, log, level, message):
        """Log a message.

        When implementing a Logger, providing this function is
        required.

        @param[in] log
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important,
            7 is least important.

        @param[in] message
            The string to be logged.

        @retval 0
            Successful."""
        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def logf(self, log, level, template, *params):
        """Log a message, constructed printf()-style.

        When implementing a Logger, providing this function is
        optional.  The interface class provides an implementation
        which formats the string, and passes it to the single-string
        variant of log().

        @param[in] log
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important, 7
            is least important.

        @param[in] template
            printf()-style template string used to create the message.

        @param[in] params
            Parameters to be substituted into the @p format string.

        @retval 0
            Successful."""

        return self.log(log, level, template % params)

class FileLog(object):
    """Simple, file-based, log implementation."""

    def __init__(self, name, directory):
        self._name = name
        self._directory = directory
        self._level = DEBUG
        self._file = None
        return

    def __del__(self):
        if self._file:
            self.close()
        return

    def set_level(self, level):
        """Set current minimum log level.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        self._level = level
        return

    def get_level(self):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        return self._level

    def open(self):
        """Open file for this log instance."""

        proc = psutil.Process(os.getpid()).name()
        filename = "flyer-%s-%s-%s.log" % (os.getlogin(), proc, self._name)
        path = os.path.join(self._directory, filename)

        self._file = open(path, "a")

        self.raw_write("SYSTEM",
                       "Opened log %s for %s (pid %u)." % (self._name,
                                                           proc,
                                                           os.getpid()))
        return

    def close(self):
        """Close and clean up this log instance."""

        if not self._file:
            return

        self.raw_write("SYSTEM", "Closing log")
        self._file.close()
        self._file = None
        return

    def log(self, level, message):
        """Log a pre-formatted message."""

        if level > self._level:
            return

        return self.raw_write(NAMES[level], message)

    def raw_write(self, level, message):
        """Write a log message to a file."""

        if not self._file:
            return

        now = time.time()
        timestamp = time.strftime("%Y-%m-%d %H:%M:%S.", time.localtime(now))
        us = (now - int(now)) * 1000000
        timestamp += "%06u" % us
        self._file.write("%s %-7s %s\n" % (timestamp, level, message))
        self._file.flush()
        return

class FileLogger(Logger):
    """Default logging implementation."""

    def __init__(self, directory):
        """Constructor."""
        super(FileLogger, self).__init__()
        self._directory = directory
        self._logs = {}
        return

    def __del__(self):
        """Destructor."""

        for name in self._logs:
            log = self._logs[name]
            log.close()

        self._logs = None
        return

    def create_log(self, name):
        """Create a new log instance.

        @param[in] name
            Name of log instance.

        @returns
           None Successful."""

        self._logs[name] = FileLog(name, self._directory)
        return

    def set_log_level(self, name, level):
        """Set current minimum log level.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        log = self._logs[name]
        return log.set_level(level)

    def get_log_level(self, name):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @param[in] name
            Name of a log instance.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        log = self._logs[name]
        return log.get_level()

    def log(self, name, level, message):
        """Log a message.

        When implementing a Logger, providing this function is
        required.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important,
            7 is least important.

        @param[in] message
            The string to be logged.

        @retval 0
            Successful."""

        log = self._logs[name]
        return log.log(level, message)
<commit_msg>Temporary work-around for those envs where os.getlogin() throws.<commit_after>#! /usr/bin/env python
#-----------------------------------------------------------------------
# COPYRIGHT_BEGIN
# Copyright (C) 2016-2017, FixFlyer, LLC.
# All rights reserved.
# COPYRIGHT_END
#-----------------------------------------------------------------------

"""Logging support."""

import os
import time
import psutil

# Emergency message level.
EMERG = 0

# Alert message level.
ALERT = 1

# Critical message level.
CRIT = 2

# Error message level.
ERR = 3

# Warning message level.
WARNING = 4

# Notice message level.
NOTICE = 5

# Informational message level.
INFO = 6

# Debugging message level.
DEBUG = 7

# Level names.
NAMES = ["EMERG", "ALERT", "CRIT", "ERR", "WARNING", "NOTICE", "INFO", "DEBUG"]

# Level numbers.
LEVELS = {"EMERG": EMERG,
          "ALERT": ALERT,
          "CRIT": CRIT,
          "ERR": ERR,
          "WARNING": WARNING,
          "NOTICE": NOTICE,
          "INFO": INFO,
          "DEBUG": DEBUG}

def get_user_name():

    user = os.environ.get('USER', os.environ.get('LOGNAME'))
    if user:
        return user

    if os.name == "posix":
        import pwd
        user = pwd.getpwuid(os.geteuid()).pw_name
        if user:
            return user

    return "unknown"

class Logger(object):
    """Logger interface.

    The API library logs informational messages describing its
    internal state and events.  By default, these messages are logged
    to a file in the host system's temporary directory.

    It's often desirable to redirect this logging to whatever facility
    is used by the client application.  The library's logging can be
    redirected by using an adaptor that implements this interface.
    """

    @staticmethod
    def string_to_level(level):
        """Convert a string level name to its integer value.

        @param[in] level
            A level name, matching the constants defined above (EMERG, etc).

        @retval @c None
            No level name matched @p level.

        @returns
            Integer level value."""

        return LEVELS.get(level)

    @staticmethod
    def level_to_string(level):
        """Convert an integer level into its string name.

        @param[in] level
            Integer level value.

        @retval None
            The supplied @p level value is invalid.

        @returns
            String name for the specified level."""

        if level < 0 or level >= len(NAMES):
            return None
        return NAMES[level]

    def create_log(self, log):
        """Create a new log instance.

        @param[in] log
            Name of log instance.

        @retval 0
           Successful."""
        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def set_log_level(self, name, level):
        """Set current minimum log level.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""

        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def get_log_level(self, name):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @param[in] name
            Name of a log instance.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""

        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def log(self, log, level, message):
        """Log a message.

        When implementing a Logger, providing this function is
        required.

        @param[in] log
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important,
            7 is least important.

        @param[in] message
            The string to be logged.

        @retval 0
            Successful."""
        # pylint: disable=unused-argument,no-self-use
        return # pragma: no cover

    def logf(self, log, level, template, *params):
        """Log a message, constructed printf()-style.

        When implementing a Logger, providing this function is
        optional.  The interface class provides an implementation
        which formats the string, and passes it to the single-string
        variant of log().

        @param[in] log
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important, 7
            is least important.

        @param[in] template
            printf()-style template string used to create the message.

        @param[in] params
            Parameters to be substituted into the @p format string.

        @retval 0
            Successful."""

        return self.log(log, level, template % params)

class FileLog(object):
    """Simple, file-based, log implementation."""

    def __init__(self, name, directory):
        self._name = name
        self._directory = directory
        self._level = DEBUG
        self._file = None
        return

    def __del__(self):
        if self._file:
            self.close()
        return

    def set_level(self, level):
        """Set current minimum log level.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        self._level = level
        return

    def get_level(self):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        return self._level

    def open(self):
        """Open file for this log instance."""

        proc = psutil.Process(os.getpid()).name()
        filename = "flyer-%s-%s-%s.log" % (get_user_name(), proc, self._name)
        path = os.path.join(self._directory, filename)

        self._file = open(path, "a")

        self.raw_write("SYSTEM",
                       "Opened log %s for %s (pid %u)." % (self._name,
                                                           proc,
                                                           os.getpid()))
        return

    def close(self):
        """Close and clean up this log instance."""

        if not self._file:
            return

        self.raw_write("SYSTEM", "Closing log")
        self._file.close()
        self._file = None
        return

    def log(self, level, message):
        """Log a pre-formatted message."""

        if level > self._level:
            return

        return self.raw_write(NAMES[level], message)

    def raw_write(self, level, message):
        """Write a log message to a file."""

        if not self._file:
            return

        now = time.time()
        timestamp = time.strftime("%Y-%m-%d %H:%M:%S.", time.localtime(now))
        us = (now - int(now)) * 1000000
        timestamp += "%06u" % us
        self._file.write("%s %-7s %s\n" % (timestamp, level, message))
        self._file.flush()
        return

class FileLogger(Logger):
    """Default logging implementation."""

    def __init__(self, directory):
        """Constructor."""
        super(FileLogger, self).__init__()
        self._directory = directory
        self._logs = {}
        return

    def __del__(self):
        """Destructor."""

        for name in self._logs:
            log = self._logs[name]
            log.close()

        self._logs = None
        return

    def create_log(self, name):
        """Create a new log instance.

        @param[in] name
            Name of log instance.

        @returns
           None Successful."""

        self._logs[name] = FileLog(name, self._directory)
        return

    def set_log_level(self, name, level):
        """Set current minimum log level.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Minimum log level of messages to be emitted.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        log = self._logs[name]
        return log.set_level(level)

    def get_log_level(self, name):
        """Get the current minimum log level.

        When logging a message, the caller specifies a level of
        importance, in the range of zero (most important) to 7 (least
        important).  Messages whose level is greater than the value
        returned from this function are not emitted.

        @param[in] name
            Name of a log instance.

        @retval 0
            Successful.

        @retval ENOENT
            @p log does not exist."""
        log = self._logs[name]
        return log.get_level()

    def log(self, name, level, message):
        """Log a message.

        When implementing a Logger, providing this function is
        required.

        @param[in] name
            Name of a log instance.

        @param[in] level
            Level of importance of this message: 0 is most important,
            7 is least important.

        @param[in] message
            The string to be logged.

        @retval 0
            Successful."""

        log = self._logs[name]
        return log.log(level, message)
<|endoftext|>


<commit_before>import hashlib
import json
import os
import tarfile
import warnings

from django.db import models
from semantic_version.django_fields import VersionField

from .constants import MODULE_REGEX
from .storage import ForgeStorage

class AuthorManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name=name)

class Author(models.Model):
    name = models.CharField(max_length=64, unique=True)

    objects = AuthorManager()

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @property
    def v3(self):
        return {
            'username': self.name.lower(),
        }

class ModuleManager(models.Manager):
    def get_by_natural_key(self, author, name):
        return self.get(author=Author.objects.get_by_natural_key(author),
                        name=name)

    def get_for_full_name(self, full_name):
        """
        Returns Module for the given full name, e.g., 'puppetlabs/stdlib'.
        """
        parsed = self.parse_full_name(full_name)
        if parsed:
            author, name = parsed
            return self.get(author__name=author, name=name)
        else:
            raise self.model.DoesNotExist

    def parse_full_name(self, full_name):
        """
        Return the module components given a full module name, or None.
        """
        match = MODULE_REGEX.match(full_name)
        if match:
            return (match.group('author'), match.group('module'))
        else:
            return None

class Module(models.Model):
    author = models.ForeignKey(Author)
    name = models.CharField(max_length=128, db_index=True)
    desc = models.TextField(db_index=True, blank=True)
    tags = models.TextField(db_index=True, blank=True)

    objects = ModuleManager()

    class Meta:
        unique_together = ('author', 'name')

    def __unicode__(self):
        return self.canonical_name

    @classmethod
    def parse_full_name(cls, full_name):
        """
        Return the module components given a full module name, or None.
        """
        warnings.warn('This classmethod is deprecated, please use the '
                      'manager method instead.', DeprecationWarning)
        return cls.objects.parse_full_name(full_name)

    @classmethod
    def get_for_full_name(cls, full_name):
        """
        Returns Module for the given full name, e.g., 'puppetlabs/stdlib'.
        """
        warnings.warn('This classmethod is deprecated, please use the '
                      'manager method instead.', DeprecationWarning)
        return cls.objects.get_for_full_name(full_name)

    @property
    def canonical_name(self):
        return u'%s-%s' % (self.author.name.lower(), self.name.lower())

    @property
    def legacy_name(self):
        return u'%s/%s' % (self.author.name.lower(), self.name.lower())

    @property
    def latest_release(self):
        """
        Return the latest version, preferably one that isn't a pre-release.
        """
        # First, try and get all non pre-release versions.
        releases = dict((release.version, release)
                        for release in self.releases.all()
                        if not release.version.prerelease)
        if not releases:
            # If all pre-releases, get all of them or return None.
            releases = dict((release.version, release)
                            for release in self.releases.all())
            if not releases:
                return None

        latest_version = max(releases.keys())
        return releases[latest_version]

    def natural_key(self):
        return (self.author.name, self.name)

    @property
    def tag_list(self):
        return self.tags.split()

    @property
    def v3_base(self):
        return {
            'name': self.name.lower(),
            'owner': self.author.v3,
        }

    @property
    def v3(self):
        v3_data = self.v3_base
        current_release = self.latest_release
        if current_release:
            current_release = current_release.v3
            v3_data.update({
                'current_release': current_release,
                'homepage_url': current_release['metadata'].get('project_page', ''),
            })
        return v3_data

def tarball_upload(instance, filename):
    author = instance.module.author.name
    return '/'.join([author[0].lower(), author, filename])

class Release(models.Model):
    module = models.ForeignKey(Module, related_name='releases')
    version = VersionField(db_index=True)
    tarball = models.FileField(upload_to=tarball_upload,
                               storage=ForgeStorage())

    class Meta:
        unique_together = ('module', 'version')

    def __unicode__(self):
        return u'%s version %s' % (self.module, self.version)

    @property
    def file_md5(self):
        # TODO: This will become an actual database field.
        self.tarball.open()
        file_md5 = hashlib.md5()
        file_md5.update(self.tarball.read())
        self.tarball.close()
        return file_md5.hexdigest()

    @property
    def file_size(self):
        return self.tarball.size

    @property
    def metadata_json(self):
        with tarfile.open(self.tarball.path, mode='r:gz') as tf:
            metadata_ti = None
            for fname in tf.getnames():
                if os.path.basename(fname) == 'metadata.json':
                    metadata_ti = tf.getmember(fname)
                    break

            if metadata_ti is None:
                raise Exception("Can't find metadata.json for release: %s" %
                                self)

            metadata = tf.extractfile(metadata_ti.name).read()

        for encoding in ('utf-8', 'latin-1'):
            try:
                return metadata.decode(encoding)
            except UnicodeDecodeError:
                continue

        raise Exception("Can't find an encoding for metadata.json")

    @property
    def metadata(self):
        return json.loads(self.metadata_json)

    @property
    def v3(self):
        return {
            'file_md5': self.file_md5,
            'file_size': self.file_size,
            'file_uri': self.tarball.url,
            'metadata': self.metadata,
            'module': self.module.v3_base,
            'tags': self.module.tag_list,
            'version': str(self.version),
        }
<commit_msg>Because we don't have `display_name` field yet, relax restrictions on case-sensitivity when getting Author/Module objects.<commit_after>import hashlib
import json
import os
import tarfile
import warnings

from django.db import models
from semantic_version.django_fields import VersionField

from .constants import MODULE_REGEX
from .storage import ForgeStorage

class AuthorManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name__iexact=name)

class Author(models.Model):
    name = models.CharField(max_length=64, unique=True)

    objects = AuthorManager()

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @property
    def v3(self):
        return {
            'username': self.name.lower(),
        }

class ModuleManager(models.Manager):
    def get_by_natural_key(self, author, name):
        return self.get(author=Author.objects.get_by_natural_key(author),
                        name__iexact=name)

    def get_for_full_name(self, full_name):
        """
        Returns Module for the given full name, e.g., 'puppetlabs/stdlib'.
        """
        parsed = self.parse_full_name(full_name)
        if parsed:
            author, name = parsed
            return self.get(author__name__iexact=author, name=name)
        else:
            raise self.model.DoesNotExist

    def parse_full_name(self, full_name):
        """
        Return the module components given a full module name, or None.
        """
        match = MODULE_REGEX.match(full_name)
        if match:
            return (match.group('author'), match.group('module'))
        else:
            return None

class Module(models.Model):
    author = models.ForeignKey(Author)
    name = models.CharField(max_length=128, db_index=True)
    desc = models.TextField(db_index=True, blank=True)
    tags = models.TextField(db_index=True, blank=True)

    objects = ModuleManager()

    class Meta:
        unique_together = ('author', 'name')

    def __unicode__(self):
        return self.canonical_name

    @classmethod
    def parse_full_name(cls, full_name):
        """
        Return the module components given a full module name, or None.
        """
        warnings.warn('This classmethod is deprecated, please use the '
                      'manager method instead.', DeprecationWarning)
        return cls.objects.parse_full_name(full_name)

    @classmethod
    def get_for_full_name(cls, full_name):
        """
        Returns Module for the given full name, e.g., 'puppetlabs/stdlib'.
        """
        warnings.warn('This classmethod is deprecated, please use the '
                      'manager method instead.', DeprecationWarning)
        return cls.objects.get_for_full_name(full_name)

    @property
    def canonical_name(self):
        return u'%s-%s' % (self.author.name.lower(), self.name.lower())

    @property
    def legacy_name(self):
        return u'%s/%s' % (self.author.name.lower(), self.name.lower())

    @property
    def latest_release(self):
        """
        Return the latest version, preferably one that isn't a pre-release.
        """
        # First, try and get all non pre-release versions.
        releases = dict((release.version, release)
                        for release in self.releases.all()
                        if not release.version.prerelease)
        if not releases:
            # If all pre-releases, get all of them or return None.
            releases = dict((release.version, release)
                            for release in self.releases.all())
            if not releases:
                return None

        latest_version = max(releases.keys())
        return releases[latest_version]

    def natural_key(self):
        return (self.author.name, self.name)

    @property
    def tag_list(self):
        return self.tags.split()

    @property
    def v3_base(self):
        return {
            'name': self.name.lower(),
            'owner': self.author.v3,
        }

    @property
    def v3(self):
        v3_data = self.v3_base
        current_release = self.latest_release
        if current_release:
            current_release = current_release.v3
            v3_data.update({
                'current_release': current_release,
                'homepage_url': current_release['metadata'].get('project_page', ''),
            })
        return v3_data

def tarball_upload(instance, filename):
    author = instance.module.author.name
    return '/'.join([author[0].lower(), author, filename])

class Release(models.Model):
    module = models.ForeignKey(Module, related_name='releases')
    version = VersionField(db_index=True)
    tarball = models.FileField(upload_to=tarball_upload,
                               storage=ForgeStorage())

    class Meta:
        unique_together = ('module', 'version')

    def __unicode__(self):
        return u'%s version %s' % (self.module, self.version)

    @property
    def file_md5(self):
        # TODO: This will become an actual database field.
        self.tarball.open()
        file_md5 = hashlib.md5()
        file_md5.update(self.tarball.read())
        self.tarball.close()
        return file_md5.hexdigest()

    @property
    def file_size(self):
        return self.tarball.size

    @property
    def metadata_json(self):
        with tarfile.open(self.tarball.path, mode='r:gz') as tf:
            metadata_ti = None
            for fname in tf.getnames():
                if os.path.basename(fname) == 'metadata.json':
                    metadata_ti = tf.getmember(fname)
                    break

            if metadata_ti is None:
                raise Exception("Can't find metadata.json for release: %s" %
                                self)

            metadata = tf.extractfile(metadata_ti.name).read()

        for encoding in ('utf-8', 'latin-1'):
            try:
                return metadata.decode(encoding)
            except UnicodeDecodeError:
                continue

        raise Exception("Can't find an encoding for metadata.json")

    @property
    def metadata(self):
        return json.loads(self.metadata_json)

    @property
    def v3(self):
        return {
            'file_md5': self.file_md5,
            'file_size': self.file_size,
            'file_uri': self.tarball.url,
            'metadata': self.metadata,
            'module': self.module.v3_base,
            'tags': self.module.tag_list,
            'version': str(self.version),
        }
<|endoftext|>