<commit_before># coding:utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2018 yutiansut/QUANTAXIS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pandas as pd
from QUANTAXIS.QAARP.QAAccount import QA_Account
from QUANTAXIS.QAUtil import (DATABASE, QA_util_log_info,
                              QA_util_random_with_topic)

class QA_Portfolio():

    """
    QUANTAXIS 多账户
    以及组合管理

    # 适用 回测/实盘

    在portfolio中,我们希望通过cookie来控制account_unit
    对于account的指标,要进行风险控制,组合成最优的投资组合的量

    用account的cookie来管理控制account
    """

    def __init__(self,user_cookie=None):
        self.accounts = {}
        self.portfolio_cookie = QA_util_random_with_topic('Portfolio')
        self.user_cookie=user_cookie
        for cookie in self.accounts.keys():
            self.accounts[cookie] = QA_Account(account_cookie=cookie)

    def __repr__(self):
        return '< QA_Portfolio {} with {} Accounts >'.format(self.portfolio_cookie, len(self.accounts.keys()))

    def get_portfolio(self):
        'return the accounts dict'
        return self.accounts

    def add_account(self, account):
        'portfolio add a account/stratetgy'
        if account.account_cookie not in self.accounts.keys():
            self.accounts[account.account_cookie] = account
        else:
            pass

    def new_account(self, account_cookie=None):
        'portfolio create a account/strategy'
        if account_cookie is None:
            temp = QA_Account(portfolio=self.portfolio_cookie,user=self.user_cookie)
            if temp.account_cookie not in self.accounts.keys():
                self.accounts[temp.account_cookie] = temp
                return temp

            else:
                return False

    def get_account(self, cookie):
        'give the account_cookie and return the account/strategy back'
        try:
            return self.accounts[cookie]
        except:
            QA_util_log_info('Can not find this account')
            return None

    def cookie_mangement(self):
        pass

    @property
    def table(self):
        return pd.concat([acc.table for acc in self.accounts.values()],axis=1)

    def get_cash(self):
        """拿到整个portfolio的可用资金

        统计每一个时间点的时候的cash总和
        """

        pass

    def pull(self, account_cookie=None, collection=DATABASE.account):
        'pull from the databases'
        if account_cookie is None:
            for item in self.accounts.keys():
                try:
                    message = collection.find_one({'cookie': item})['message']
                    QA_util_log_info('{} sync successfully'.format(item))
                except Exception as e:
                    QA_util_log_info(
                        '{} sync wrong \\\n wrong info {}'.format(item, e))
                self.accounts[item].from_message(message)

        else:
            try:
                message = collection.find_one(
                    {'cookie': account_cookie})['message']
                QA_util_log_info('{} sync successfully'.format(item))
            except Exception as e:
                QA_util_log_info(
                    '{} sync wrong \\\n wrong info {}'.format(account_cookie, e))
            self.accounts[account_cookie].from_message(message)

    def push(self, account_cookie=None, collection=DATABASE.account):
        'push to databases'
        message = self.accounts[account_cookie].message
        if account_cookie is None:
            for item in self.accounts.keys():
                try:
                    message = collection.find_one_and_update({'cookie': item})
                    QA_util_log_info('{} sync successfully'.format(item))
                except Exception as e:
                    QA_util_log_info(
                        '{} sync wrong \\\n wrong info {}'.format(item, e))
                self.accounts[item].from_message(message)

        else:
            try:
                message = collection.find_one(
                    {'cookie': account_cookie})['message']
                QA_util_log_info('{} sync successfully'.format(item))
            except Exception as e:
                QA_util_log_info(
                    '{} sync wrong \\\n wrong info {}'.format(account_cookie, e))
            self.accounts[account_cookie].from_message(message)
<commit_msg>#commit for portfolio.add_account<commit_after># coding:utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2018 yutiansut/QUANTAXIS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pandas as pd
from QUANTAXIS.QAARP.QAAccount import QA_Account
from QUANTAXIS.QAUtil import (DATABASE, QA_util_log_info,
                              QA_util_random_with_topic)

class QA_Portfolio():

    """
    QUANTAXIS 多账户
    以及组合管理

    # 适用 回测/实盘

    在portfolio中,我们希望通过cookie来控制account_unit
    对于account的指标,要进行风险控制,组合成最优的投资组合的量

    用account的cookie来管理控制account
    """

    def __init__(self, user_cookie=None):
        self.accounts = {}
        self.portfolio_cookie = QA_util_random_with_topic('Portfolio')
        self.user_cookie = user_cookie
        for cookie in self.accounts.keys():
            self.accounts[cookie] = QA_Account(account_cookie=cookie)

    def __repr__(self):
        return '< QA_Portfolio {} with {} Accounts >'.format(self.portfolio_cookie, len(self.accounts.keys()))

    def get_portfolio(self):
        'return the accounts dict'
        return self.accounts

    def add_account(self, account):
        'portfolio add a account/stratetgy'
        if account.account_cookie not in self.accounts.keys():
            account.portfolio = self.portfolio_cookie
            account.user = self.user_cookie
            self.accounts[account.account_cookie] = account
        else:
            pass

    def new_account(self, account_cookie=None):
        'portfolio create a account/strategy'
        if account_cookie is None:
            temp = QA_Account(portfolio=self.portfolio_cookie,
                              user=self.user_cookie)
            if temp.account_cookie not in self.accounts.keys():
                self.accounts[temp.account_cookie] = temp
                return temp

            else:
                return False

    def get_account(self, cookie):
        'give the account_cookie and return the account/strategy back'
        try:
            return self.accounts[cookie]
        except:
            QA_util_log_info('Can not find this account')
            return None

    def cookie_mangement(self):
        pass

    @property
    def table(self):
        return pd.concat([acc.table for acc in self.accounts.values()], axis=1)

    def get_cash(self):
        """拿到整个portfolio的可用资金

        统计每一个时间点的时候的cash总和
        """

        pass

    def pull(self, account_cookie=None, collection=DATABASE.account):
        'pull from the databases'
        if account_cookie is None:
            for item in self.accounts.keys():
                try:
                    message = collection.find_one({'cookie': item})['message']
                    QA_util_log_info('{} sync successfully'.format(item))
                except Exception as e:
                    QA_util_log_info(
                        '{} sync wrong \\\n wrong info {}'.format(item, e))
                self.accounts[item].from_message(message)

        else:
            try:
                message = collection.find_one(
                    {'cookie': account_cookie})['message']
                QA_util_log_info('{} sync successfully'.format(item))
            except Exception as e:
                QA_util_log_info(
                    '{} sync wrong \\\n wrong info {}'.format(account_cookie, e))
            self.accounts[account_cookie].from_message(message)

    def push(self, account_cookie=None, collection=DATABASE.account):
        'push to databases'
        message = self.accounts[account_cookie].message
        if account_cookie is None:
            for item in self.accounts.keys():
                try:
                    message = collection.find_one_and_update({'cookie': item})
                    QA_util_log_info('{} sync successfully'.format(item))
                except Exception as e:
                    QA_util_log_info(
                        '{} sync wrong \\\n wrong info {}'.format(item, e))
                self.accounts[item].from_message(message)

        else:
            try:
                message = collection.find_one(
                    {'cookie': account_cookie})['message']
                QA_util_log_info('{} sync successfully'.format(item))
            except Exception as e:
                QA_util_log_info(
                    '{} sync wrong \\\n wrong info {}'.format(account_cookie, e))
            self.accounts[account_cookie].from_message(message)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

def get_longest_common_prefix(l):
    def get_common_prefix(a, b):
        if not a or not b:
            return ''
        steps = min(len(a), len(b))
        result = ''
        index = 0
        while index < steps:
            if a[index] == b[index]:
                result += a[index]
            else:
                return result
            index += 1
        return result

    return reduce(get_common_prefix, l)

if __name__ == '__main__':
    result = get_longest_common_prefix(["abcdef", "abcdk", "abcfk"])
    print(result)
<commit_msg>finish day36: get longest common prefix.<commit_after># -*- coding:utf-8 -*-
"""
看了你的逻辑,还是你的时间复杂度低一点儿O(m*n), m为字符串个数,n为公共子串长度
我的是先求第一个字符串和第二个字符串的的公共子串a,然后a再与第三个字符串像比求得公共子串c...依次类推.时间复杂度肯定要比你的高.
"""

def get_longest_common_prefix(l):
    def get_common_prefix(a, b):
        if not a or not b:
            return ''
        steps = min(len(a), len(b))
        result = ''
        index = 0
        while index < steps:
            if a[index] == b[index]:
                result += a[index]
            else:
                return result
            index += 1
        return result

    return reduce(get_common_prefix, l)

if __name__ == '__main__':
    result = get_longest_common_prefix(["abcdef", "abcdk", "abcfk"])
    print(result)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Database schema and data updates.

    :copyright: Copyright (c) 2014 Bivio Software, Inc.  All Rights Reserved.
    :license: Apache, see LICENSE for more details.
"""

import flask_script as fes
from publicprize.controller import db
import publicprize.auth.model as pam
import publicprize.controller as ppc
import publicprize.nextup.model as pnm

# Needs to be explicit
ppc.init()
_MANAGER = fes.Manager(ppc.app())

@_MANAGER.command
def clear_all_sessions():
    """Logs out all users - clears all user session state from db."""
    db.get_engine(ppc.app()).execute('DELETE FROM beaker_cache')

@_MANAGER.command
def upgrade_judge_rank_table():
    pnm.JudgeRank.__table__.create(bind=db.get_engine(ppc.app()))

@_MANAGER.command
def upgrade_numodel_table():
    _add_column(pnm.NUContest, pnm.NUContest.end_date, "'2015-05-08'")

def _add_column(model, column, default_value=None):
    """Adds the column to the database. Sets default_value if column
    is not nullable."""
    engine = db.get_engine(ppc.app())
    colname = column.description
    coltype = column.type.compile(engine.dialect)
    table = model.__table__.description
    engine.execute(
        'ALTER TABLE {} ADD COLUMN {} {}'.format(table, colname, coltype))
    if not column.nullable:
        if default_value is None:
            raise Exception('NOT_NULL column missing default value')
        engine.execute(
            'UPDATE {} SET {} = {}'.format(table, colname, default_value))
        engine.execute(
            'ALTER TABLE {} ALTER COLUMN {} SET NOT NULL'.format(
                table, colname))

def _add_enum_type(type_name, values):
    """Adds an enum type to the database."""
    #CREATE TYPE bug_status AS ENUM ('new', 'open', 'closed');
    db.get_engine(ppc.app()).execute('CREATE TYPE {} AS ENUM ({})'.format(
            type_name,
            ','.join(list(map((lambda x: "'{}'".format(x)), values)))))

if __name__ == '__main__':
    _MANAGER.run()
<commit_msg>commented out clear_all_sessions()<commit_after># -*- coding: utf-8 -*-
""" Database schema and data updates.

    :copyright: Copyright (c) 2014 Bivio Software, Inc.  All Rights Reserved.
    :license: Apache, see LICENSE for more details.
"""

import flask_script as fes
from publicprize.controller import db
import publicprize.auth.model as pam
import publicprize.controller as ppc
import publicprize.nextup.model as pnm

# Needs to be explicit
ppc.init()
_MANAGER = fes.Manager(ppc.app())

# @_MANAGER.command
# def clear_all_sessions():
#     """Logs out all users - clears all user session state from db."""
#     db.get_engine(ppc.app()).execute('DELETE FROM beaker_cache')

@_MANAGER.command
def upgrade_judge_rank_table():
    pnm.JudgeRank.__table__.create(bind=db.get_engine(ppc.app()))

@_MANAGER.command
def upgrade_numodel_table():
    _add_column(pnm.NUContest, pnm.NUContest.end_date, "'2015-05-08'")

def _add_column(model, column, default_value=None):
    """Adds the column to the database. Sets default_value if column
    is not nullable."""
    engine = db.get_engine(ppc.app())
    colname = column.description
    coltype = column.type.compile(engine.dialect)
    table = model.__table__.description
    engine.execute(
        'ALTER TABLE {} ADD COLUMN {} {}'.format(table, colname, coltype))
    if not column.nullable:
        if default_value is None:
            raise Exception('NOT_NULL column missing default value')
        engine.execute(
            'UPDATE {} SET {} = {}'.format(table, colname, default_value))
        engine.execute(
            'ALTER TABLE {} ALTER COLUMN {} SET NOT NULL'.format(
                table, colname))

def _add_enum_type(type_name, values):
    """Adds an enum type to the database."""
    #CREATE TYPE bug_status AS ENUM ('new', 'open', 'closed');
    db.get_engine(ppc.app()).execute('CREATE TYPE {} AS ENUM ({})'.format(
            type_name,
            ','.join(list(map((lambda x: "'{}'".format(x)), values)))))

if __name__ == '__main__':
    _MANAGER.run()
<|endoftext|>


<commit_before>import pytest, time, logging
from selenium import webdriver
#logging.basicConfig(format = u'%(filename)s[LINE:%(lineno)d]# %(levelname)-8s [%(asctime)s]  %(message)s', level = logging.DEBUG)
logging.basicConfig(filename="sample.log", level=logging.INFO)

logging.debug("This is a debug message")
logging.info("Informational message")
logging.error("An error has happened!")

@pytest.yield_fixture()
def browser(request):
    driver = webdriver.Chrome()
    yield driver
    driver.quit()

def test_python_org(browser):
    browser.get('http://python.org')
    assert 'Welcome to Python.org' in browser.title
    browser.find_element_by_css_selector("#id-search-field").send_keys("pycon")
    browser.find_element_by_name("submit").click()
    browser.find_element_by_css_selector("#content > div > section > form > ul > li:nth-child(1) > h3 > a").click()
    #time.sleep(10)
<commit_msg>Добавил описание<commit_after>#######################
#Pytest fixture test sample
#######################

import pytest, time, logging
from selenium import webdriver
#logging.basicConfig(format = u'%(filename)s[LINE:%(lineno)d]# %(levelname)-8s [%(asctime)s]  %(message)s', level = logging.DEBUG)
logging.basicConfig(filename="sample.log", level=logging.INFO)

logging.debug("This is a debug message")
logging.info("Informational message")
logging.error("An error has happened!")

@pytest.yield_fixture()
def browser(request):
    driver = webdriver.Chrome()
    yield driver
    driver.quit()

def test_python_org(browser):
    browser.get('http://python.org')
    assert 'Welcome to Python.org' in browser.title
    browser.find_element_by_css_selector("#id-search-field").send_keys("pycon")
    browser.find_element_by_name("submit").click()
    browser.find_element_by_css_selector("#content > div > section > form > ul > li:nth-child(1) > h3 > a").click()
    #time.sleep(10)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  trans_tests.py:  testing eol conversion and keyword substitution
#
#  Subversion is a tool for revision control.
#  See http://subversion.tigris.org for more information.
#
# ====================================================================
# Copyright (c) 2000-2001 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

# General modules
import shutil, string, sys, re, os.path, traceback

# The `svntest' module
try:
  import svntest
except SyntaxError:
  sys.stderr.write('[SKIPPED] ')
  print "<<< Please make sure you have Python 2 or better! >>>"
  traceback.print_exc(None,sys.stdout)
  raise SystemExit

# (abbreviation)
path_index = svntest.actions.path_index

######################################################################
# THINGS TO TEST
#
# *** Perhaps something like commit_tests.py:make_standard_slew_of_changes
#     is in order here in this file as well. ***
#
# status level 1:
#    enable translation, status
#    (now throw local text mods into the picture)
#
# commit level 1:
#    enable translation, commit
#    (now throw local text mods into the picture)
#
# checkout:
#    checkout stuff with translation enabled
#
# status level 2:
#    disable translation, status
#    change newline conversion to different style, status
#    (now throw local text mods into the picture)
#
# commit level 2:
#    disable translation, commit
#    change newline conversion to different style, commit
#    (now throw local text mods into the picture)
#    (now throw local text mods with tortured line endings into the picture)
#
# update:
#    update files from disabled translation to enabled translation
#    update files from enabled translation to disabled translation
#    update files with newline conversion style changes
#    (now throw local text mods into the picture)
#    (now throw conflicting local property mods into the picture)
#
####

########### THINGS THAT HAVE FAILED DURING HAND-TESTING ##############
#
# These have all been fixed, but we want regression tests for them.
#
#   1. Ben encountered this:
#      Create a greek tree, commit a keyword into one file,
#      then commit a keyword property (i.e., turn on keywords), then
#      try to check out head somewhere else.  See seg fault.
#
#   2. Mike encountered this:
#      Add the keyword property to a file, svn revert the file, see
#      error.
#
#   3. Another one from Ben:
#      Keywords not expanded on checkout.
#
######################################################################

# Paths that the tests test.
author_rev_unexp_path = ''
author_rev_exp_path = ''
bogus_keywords_path = ''
embd_author_rev_unexp_path = ''
embd_author_rev_exp_path = ''
embd_bogus_keywords_path = ''

def setup_working_copy(wc_dir):
  """Setup a standard test working copy, then create (but do not add)
  various files for testing translation."""

  global author_rev_unexp_path
  global author_rev_exp_path
  global bogus_keywords_path
  global embd_author_rev_unexp_path
  global embd_author_rev_exp_path
  global embd_bogus_keywords_path

  # NOTE: Only using author and revision keywords in tests for now,
  # since they return predictable substitutions.

  # Unexpanded, expanded, and bogus keywords; first as the only
  # contents of the files, then embedded in non-keyword content.
  author_rev_unexp_path = os.path.join(wc_dir, 'author_rev_unexp')
  author_rev_exp_path = os.path.join(wc_dir, 'author_rev_exp')
  bogus_keywords_path = os.path.join(wc_dir, 'bogus_keywords')
  embd_author_rev_unexp_path = os.path.join(wc_dir, 'embd_author_rev_unexp')
  embd_author_rev_exp_path = os.path.join(wc_dir, 'embd_author_rev_exp')
  embd_bogus_keywords_path = os.path.join(wc_dir, 'embd_bogus_keywords')

  svntest.main.file_append (author_rev_unexp_path, "$Author$\n$Rev$")
  svntest.main.file_append (author_rev_exp_path, "$Author: blah $\n$Rev: 0 $")
  svntest.main.file_append (bogus_keywords_path, "$Arthur$\n$Rev0$")
  svntest.main.file_append (embd_author_rev_unexp_path,
                            "one\nfish\n$Author$ two fish\n red $Rev$\n fish")
  svntest.main.file_append (embd_author_rev_exp_path,
                            "blue $Author: blah $ fish$Rev: 0 $\nI fish")
  svntest.main.file_append (embd_bogus_keywords_path,
                            "you fish $Arthur$then\n we$Rev0$ \n\nchew fish")

  return 0

### Helper functions for setting/removing properties

# Set the property keyword for PATH.  Turn on all possible keywords.
# ### todo: Later, take list of keywords to set.
def keywords_on(path):
  svntest.main.run_svn(None, 'propset',
                       "svn:keywords", "Author Rev Date URL", path)

# Delete property NAME from versioned PATH in the working copy.
# ### todo: Later, take list of keywords to remove from the propval?
def keywords_off(path):
  svntest.main.run_svn(None, 'propdel', "svn:keywords", path)

######################################################################
# Tests
#
#   Each test must return 0 on success or non-zero on failure.

#----------------------------------------------------------------------

def keywords_from_birth(sbox):
  """Create some files that have the `svn:keywords' property set from
  the moment they're first committed.  Some of the files actually
  contain keywords, others don't.  Make sure everything behaves
  correctly."""

  if sbox.build():
    return 1

  wc_dir = sbox.wc_dir

  if setup_working_copy (wc_dir): return 1

  keywords_on (author_rev_unexp_path)
  keywords_on (embd_author_rev_exp_path)

  # Add all the files
  status_list = svntest.actions.get_virginal_status_list (wc_dir, '1')
  status_list.append ([author_rev_unexp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([author_rev_exp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([bogus_keywords_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_author_rev_unexp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_author_rev_exp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_bogus_keywords_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])

  expected_output_tree = svntest.tree.build_generic_tree (status_list)

  svntest.main.run_svn (None, 'add', author_rev_unexp_path)
  svntest.main.run_svn (None, 'add', author_rev_exp_path)
  svntest.main.run_svn (None, 'add', bogus_keywords_path)
  svntest.main.run_svn (None, 'add', embd_author_rev_unexp_path)
  svntest.main.run_svn (None, 'add', embd_author_rev_exp_path)
  svntest.main.run_svn (None, 'add', embd_bogus_keywords_path)

  svntest.actions.run_and_verify_status(wc_dir, expected_output_tree)

  # Commit.
  output_list = [[author_rev_unexp_path, None, {}, {'verb' : 'Sending' }],
                 [author_rev_exp_path, None, {}, {'verb' : 'Sending' }],
                 [bogus_keywords_path, None, {}, {'verb' : 'Sending' }],
                 [embd_author_rev_unexp_path, None, {}, {'verb' : 'Sending' }],
                 [embd_author_rev_exp_path, None, {}, {'verb' : 'Sending' }],
                 [embd_bogus_keywords_path, None, {}, {'verb' : 'Sending' }]]
  expected_output_tree = svntest.tree.build_generic_tree(output_list)

  if svntest.actions.run_and_verify_commit (wc_dir, expected_output_tree,
                                            None, None,
                                            None, None, None, None, wc_dir):
    return 1

  return 0

def enable_translation(sbox):
  "enable translation, check status, commit"

  wc_dir = sbox.wc_dir

  # TODO: Turn on newline conversion and/or keyword substition for all
  # sorts of files, with and without local mods, and verify that
  # status shows the right stuff.  The, commit those mods.

  return 0

#----------------------------------------------------------------------

def checkout_translated():
  "checkout files that have translation enabled"

  # TODO: Checkout a tree which contains files with translation
  # enabled.

  return 0

#----------------------------------------------------------------------

def disable_translation():
  "disable translation, check status, commit"

  # TODO: Disable translation on files which have had it enabled,
  # with and without local mods, check status, and commit.

  return 0

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              keywords_from_birth,
              enable_translation,
              checkout_translated,
              disable_translation,
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
# local variables:
# eval: (load-file "../../../svn-dev.el")
# end:
<commit_msg><commit_after>#!/usr/bin/env python
#
#  trans_tests.py:  testing eol conversion and keyword substitution
#
#  Subversion is a tool for revision control.
#  See http://subversion.tigris.org for more information.
#
# ====================================================================
# Copyright (c) 2000-2001 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################

# General modules
import shutil, string, sys, re, os.path, traceback

# The `svntest' module
try:
  import svntest
except SyntaxError:
  sys.stderr.write('[SKIPPED] ')
  print "<<< Please make sure you have Python 2 or better! >>>"
  traceback.print_exc(None,sys.stdout)
  raise SystemExit

# (abbreviation)
path_index = svntest.actions.path_index

######################################################################
# THINGS TO TEST
#
# *** Perhaps something like commit_tests.py:make_standard_slew_of_changes
#     is in order here in this file as well. ***
#
# status level 1:
#    enable translation, status
#    (now throw local text mods into the picture)
#
# commit level 1:
#    enable translation, commit
#    (now throw local text mods into the picture)
#
# checkout:
#    checkout stuff with translation enabled
#
# status level 2:
#    disable translation, status
#    change newline conversion to different style, status
#    (now throw local text mods into the picture)
#
# commit level 2:
#    disable translation, commit
#    change newline conversion to different style, commit
#    (now throw local text mods into the picture)
#    (now throw local text mods with tortured line endings into the picture)
#
# update:
#    update files from disabled translation to enabled translation
#    update files from enabled translation to disabled translation
#    update files with newline conversion style changes
#    (now throw local text mods into the picture)
#    (now throw conflicting local property mods into the picture)
#
####

########### THINGS THAT HAVE FAILED DURING HAND-TESTING ##############
#
# These have all been fixed, but we want regression tests for them.
#
#   1. Ben encountered this:
#      Create a greek tree, commit a keyword into one file,
#      then commit a keyword property (i.e., turn on keywords), then
#      try to check out head somewhere else.  See seg fault.
#
#   2. Mike encountered this:
#      Add the keyword property to a file, svn revert the file, see
#      error.
#
#   3. Another one from Ben:
#      Keywords not expanded on checkout.
#
######################################################################

# Paths that the tests test.
author_rev_unexp_path = ''
author_rev_exp_path = ''
bogus_keywords_path = ''
embd_author_rev_unexp_path = ''
embd_author_rev_exp_path = ''
embd_bogus_keywords_path = ''

def setup_working_copy(wc_dir):
  """Setup a standard test working copy, then create (but do not add)
  various files for testing translation."""

  global author_rev_unexp_path
  global author_rev_exp_path
  global bogus_keywords_path
  global embd_author_rev_unexp_path
  global embd_author_rev_exp_path
  global embd_bogus_keywords_path

  # NOTE: Only using author and revision keywords in tests for now,
  # since they return predictable substitutions.

  # Unexpanded, expanded, and bogus keywords; first as the only
  # contents of the files, then embedded in non-keyword content.
  author_rev_unexp_path = os.path.join(wc_dir, 'author_rev_unexp')
  author_rev_exp_path = os.path.join(wc_dir, 'author_rev_exp')
  bogus_keywords_path = os.path.join(wc_dir, 'bogus_keywords')
  embd_author_rev_unexp_path = os.path.join(wc_dir, 'embd_author_rev_unexp')
  embd_author_rev_exp_path = os.path.join(wc_dir, 'embd_author_rev_exp')
  embd_bogus_keywords_path = os.path.join(wc_dir, 'embd_bogus_keywords')

  svntest.main.file_append (author_rev_unexp_path, "$Author$\n$Rev$")
  svntest.main.file_append (author_rev_exp_path, "$Author: blah $\n$Rev: 0 $")
  svntest.main.file_append (bogus_keywords_path, "$Arthur$\n$Rev0$")
  svntest.main.file_append (embd_author_rev_unexp_path,
                            "one\nfish\n$Author$ two fish\n red $Rev$\n fish")
  svntest.main.file_append (embd_author_rev_exp_path,
                            "blue $Author: blah $ fish$Rev: 0 $\nI fish")
  svntest.main.file_append (embd_bogus_keywords_path,
                            "you fish $Arthur$then\n we$Rev0$ \n\nchew fish")

  return 0

### Helper functions for setting/removing properties

# Set the property keyword for PATH.  Turn on all possible keywords.
# ### todo: Later, take list of keywords to set.
def keywords_on(path):
  svntest.main.run_svn(None, 'propset',
                       "svn:keywords", "Author Rev Date URL", path)

# Delete property NAME from versioned PATH in the working copy.
# ### todo: Later, take list of keywords to remove from the propval?
def keywords_off(path):
  svntest.main.run_svn(None, 'propdel', "svn:keywords", path)

######################################################################
# Tests
#
#   Each test must return 0 on success or non-zero on failure.

#----------------------------------------------------------------------

def keywords_from_birth(sbox):
  "Commit new files with keywords active from birth"

  if sbox.build():
    return 1

  wc_dir = sbox.wc_dir

  if setup_working_copy (wc_dir): return 1

  # Add all the files
  status_list = svntest.actions.get_virginal_status_list (wc_dir, '1')
  status_list.append ([author_rev_unexp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([author_rev_exp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([bogus_keywords_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_author_rev_unexp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_author_rev_exp_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])
  status_list.append ([embd_bogus_keywords_path, None, {},
                       {'status' : 'A ',
                        'wc_rev' : '0',
                        'repos_rev' : '1'}])

  expected_output_tree = svntest.tree.build_generic_tree (status_list)

  svntest.main.run_svn (None, 'add', author_rev_unexp_path)
  svntest.main.run_svn (None, 'add', author_rev_exp_path)
  svntest.main.run_svn (None, 'add', bogus_keywords_path)
  svntest.main.run_svn (None, 'add', embd_author_rev_unexp_path)
  svntest.main.run_svn (None, 'add', embd_author_rev_exp_path)
  svntest.main.run_svn (None, 'add', embd_bogus_keywords_path)

  svntest.actions.run_and_verify_status(wc_dir, expected_output_tree)

  # Add the keyword properties.
  keywords_on (author_rev_unexp_path)
  keywords_on (embd_author_rev_exp_path)

  # Commit.
  output_list = [[author_rev_unexp_path, None, {}, {'verb' : 'Adding' }],
                 [author_rev_exp_path, None, {}, {'verb' : 'Adding' }],
                 [bogus_keywords_path, None, {}, {'verb' : 'Adding' }],
                 [embd_author_rev_unexp_path, None, {}, {'verb' : 'Adding' }],
                 [embd_author_rev_exp_path, None, {}, {'verb' : 'Adding' }],
                 [embd_bogus_keywords_path, None, {}, {'verb' : 'Adding' }]]
  expected_output_tree = svntest.tree.build_generic_tree(output_list)

  if svntest.actions.run_and_verify_commit (wc_dir, expected_output_tree,
                                            None, None,
                                            None, None, None, None, wc_dir):
    return 1

  return 0

def enable_translation(sbox):
  "enable translation, check status, commit"

  wc_dir = sbox.wc_dir

  # TODO: Turn on newline conversion and/or keyword substition for all
  # sorts of files, with and without local mods, and verify that
  # status shows the right stuff.  The, commit those mods.

  return 0

#----------------------------------------------------------------------

def checkout_translated():
  "checkout files that have translation enabled"

  # TODO: Checkout a tree which contains files with translation
  # enabled.

  return 0

#----------------------------------------------------------------------

def disable_translation():
  "disable translation, check status, commit"

  # TODO: Disable translation on files which have had it enabled,
  # with and without local mods, check status, and commit.

  return 0

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              keywords_from_birth,
              enable_translation,
              checkout_translated,
              disable_translation,
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
# local variables:
# eval: (load-file "../../../svn-dev.el")
# end:
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  svndumpfilter_tests.py:  testing the 'svndumpfilter' tool.
#
#  Subversion is a tool for revision control.
#  See http://subversion.apache.org for more information.
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
######################################################################

# General modules
import os
import sys
import tempfile

# Our testing module
import svntest
from svntest.verify import SVNExpectedStdout, SVNExpectedStderr

# Get some helper routines from svnadmin_tests
from svnadmin_tests import load_and_verify_dumpstream, test_create

# (abbreviation)
Skip = svntest.testcase.Skip
SkipUnless = svntest.testcase.SkipUnless
XFail = svntest.testcase.XFail
Item = svntest.wc.StateItem

######################################################################
# Helper routines

def filter_and_return_output(dump, bufsize=0, *varargs):
  """Filter the array of lines passed in 'dump' and return the output"""

  if isinstance(dump, str):
    dump = [ dump ]

  ## TODO: Should we need to handle errput and exit_code?
  exit_code, output, errput = svntest.main.run_command_stdin(
    svntest.main.svndumpfilter_binary, None, bufsize, 1, dump, *varargs)

  return output

######################################################################
# Tests

def reflect_dropped_renumbered_revs(sbox):
  "reflect dropped renumbered revs in svn:mergeinfo"

  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=2982. ##

  # Test svndumpfilter with include option
  test_create(sbox)
  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'with_merges.dump')
  dumpfile = open(dumpfile_location).read()

  filtered_out = filter_and_return_output(dumpfile, 0, "include",
                                          "trunk", "branch1",
                                          "--skip-missing-merge-sources",
                                          "--drop-empty-revs",
                                          "--renumber-revs", "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_out,
                             "--ignore-uuid")

  # Verify the svn:mergeinfo properties
  url = sbox.repo_url
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - /branch1:4-5\n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

  # Test svndumpfilter with exclude option
  test_create(sbox)
  filtered_out = filter_and_return_output(dumpfile, 0, "exclude",
                                          "branch1",
                                          "--skip-missing-merge-sources",
                                          "--drop-empty-revs",
                                          "--renumber-revs", "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_out,
                             "--ignore-uuid")

  # Verify the svn:mergeinfo properties
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - \n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

def svndumpfilter_loses_mergeinfo(sbox):
  "svndumpfilter loses mergeinfo"
  #svndumpfilter loses mergeinfo if invoked without --renumber-revs

  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=3181. ##

  test_create(sbox)
  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'with_merges.dump')
  dumpfile = open(dumpfile_location).read()

  filtered_out = filter_and_return_output(dumpfile, 0, "include",
                                          "trunk", "branch1", "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_out)

  # Verify the svn:mergeinfo properties
  url = sbox.repo_url
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - /branch1:4-8\n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

def _simple_dumpfilter_test(sbox, dumpfile, *dumpargs):
  wc_dir = sbox.wc_dir

  filtered_output = filter_and_return_output(dumpfile, 0,
                                             '--quiet', *dumpargs)

  # Setup our expectations
  load_and_verify_dumpstream(sbox, [], [], None, filtered_output,
                             '--ignore-uuid')
  expected_disk = svntest.main.greek_state.copy()
  expected_disk.remove('A/B/E/alpha')
  expected_disk.remove('A/B/E/beta')
  expected_disk.remove('A/B/E')
  expected_disk.remove('A/D/H/chi')
  expected_disk.remove('A/D/H/psi')
  expected_disk.remove('A/D/H/omega')
  expected_disk.remove('A/D/H')
  expected_disk.remove('A/D/G/pi')
  expected_disk.remove('A/D/G/rho')
  expected_disk.remove('A/D/G/tau')
  expected_disk.remove('A/D/G')

  expected_output = svntest.wc.State(wc_dir, {
    'A'           : Item(status='A '),
    'A/B'         : Item(status='A '),
    'A/B/lambda'  : Item(status='A '),
    'A/B/F'       : Item(status='A '),
    'A/mu'        : Item(status='A '),
    'A/C'         : Item(status='A '),
    'A/D'         : Item(status='A '),
    'A/D/gamma'   : Item(status='A '),
    'iota'        : Item(status='A '),
    })

  expected_status = svntest.actions.get_virginal_state(wc_dir, 1)
  expected_status.remove('A/B/E/alpha')
  expected_status.remove('A/B/E/beta')
  expected_status.remove('A/B/E')
  expected_status.remove('A/D/H/chi')
  expected_status.remove('A/D/H/psi')
  expected_status.remove('A/D/H/omega')
  expected_status.remove('A/D/H')
  expected_status.remove('A/D/G/pi')
  expected_status.remove('A/D/G/rho')
  expected_status.remove('A/D/G/tau')
  expected_status.remove('A/D/G')

  # Check that our paths really were excluded
  svntest.actions.run_and_verify_update(wc_dir,
                                        expected_output,
                                        expected_disk,
                                        expected_status)

def dumpfilter_with_targets(sbox):
  "svndumpfilter --targets blah"
  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=2697. ##

  test_create(sbox)

  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'greek_tree.dump')
  dumpfile = open(dumpfile_location).read()

  (fd, targets_file) = tempfile.mkstemp(dir=svntest.main.temp_dir)
  try:
    targets = open(targets_file, 'w')
    targets.write('/A/D/H\n')
    targets.write('/A/D/G\n')
    targets.close()
    _simple_dumpfilter_test(sbox, dumpfile,
                            'exclude', '/A/B/E', '--targets', targets_file)
  finally:
    os.close(fd)
    os.remove(targets_file)

def dumpfilter_with_patterns(sbox):
  "svndumpfilter --pattern PATH_PREFIX"

  test_create(sbox)

  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'greek_tree.dump')
  dumpfile = open(dumpfile_location).read()
  _simple_dumpfilter_test(sbox, dumpfile,
                          'exclude', '--pattern', '/A/D/[GH]*', '/A/[B]/E*')

#----------------------------------------------------------------------
# More testing for issue #3020 'Reflect dropped/renumbered revisions in
# svn:mergeinfo data during svnadmin load'
#
# Specifically, test that svndumpfilter, when used with the
# --skip-missing-merge-sources option, removes mergeinfo that refers to
# revisions that are older than the oldest revision in the dump stream.
def filter_mergeinfo_revs_outside_of_dump_stream(sbox):
  "filter mergeinfo revs outside of dump stream"

  test_create(sbox)

  # Load a partial dump into an existing repository.
  #
  # Picture == 1k words:
  #
  # The dump file we filter in this test, 'mergeinfo_included_partial.dump', is
  # a dump of r6:HEAD of the following repos:
  #
  #                       __________________________________________
  #                      |                                         |
  #                      |             ____________________________|_____
  #                      |            |                            |     |
  # trunk---r2---r3-----r5---r6-------r8---r9--------------->      |     |
  #   r1             |        |     |       |                      |     |
  # intial           |        |     |       |______                |     |
  # import         copy       |   copy             |            merge   merge
  #                  |        |     |            merge           (r5)   (r8)
  #                  |        |     |            (r9)              |     |
  #                  |        |     |              |               |     |
  #                  |        |     V              V               |     |
  #                  |        | branches/B2-------r11---r12---->   |     |
  #                  |        |     r7              |____|         |     |
  #                  |        |                        |           |     |
  #                  |      merge                      |___        |     |
  #                  |      (r6)                           |       |     |
  #                  |        |_________________           |       |     |
  #                  |                          |        merge     |     |
  #                  |                          |      (r11-12)    |     |
  #                  |                          |          |       |     |
  #                  V                          V          V       |     |
  #              branches/B1-------------------r10--------r13-->   |     |
  #                  r4                                            |     |
  #                   |                                            V     V
  #                  branches/B1/B/E------------------------------r14---r15->
  #
  #
  # The mergeinfo on the complete repos would look like this:
  #
  #   Properties on 'branches/B1':
  #     svn:mergeinfo
  #       /branches/B2:11-12
  #       /trunk:6,9
  #   Properties on 'branches/B1/B/E':
  #     svn:mergeinfo
  #       /branches/B2/B/E:11-12
  #       /trunk/B/E:5-6,8-9
  #   Properties on 'branches/B2':
  #     svn:mergeinfo
  #       /trunk:9
  #
  # We will run the partial dump through svndumpfilter using the the
  # --skip-missing-merge-soruces which should strip out any revisions < 6.
  # Then we'll load the filtered result into an empty repository.  This
  # should offset the incoming mergeinfo by -5.  The resulting mergeinfo
  # should look like this:
  #
  #   Properties on 'branches/B1':
  #     svn:mergeinfo
  #       /branches/B2:6-7
  #       /trunk:1,4
  #   Properties on 'branches/B1/B/E':
  #     svn:mergeinfo
  #       /branches/B2/B/E:6-7
  #       /trunk/B/E:1,3-4
  #   Properties on 'branches/B2':
  #     svn:mergeinfo
  #       /trunk:4
  partial_dump = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'mergeinfo_included_partial.dump')
  partial_dump_contents = open(partial_dump).read()
  filtered_dumpfile2 = filter_and_return_output(partial_dump_contents,
                                                8192, # Set a sufficiently
                                                      # large bufsize to
                                                      # avoid a deadlock
                                                "include", "trunk", "branches",
                                                "--skip-missing-merge-sources",
                                                "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_dumpfile2,
                             '--ignore-uuid')
  # Check the resulting mergeinfo.
  #
  # Currently this fails with this mergeinfo:
  #
  #   Properties on 'branches\B1':
  #     svn:mergeinfo
  #       /branches/B2:6-7
  #       /trunk:4,5-2
  #                ^^^
  # What's this?!?!  It's corrupt mergeinfo caused when svnadmin load
  # maps the end rev of the incoming 5-6, but not the start rev, see '2)'
  # in http://subversion.tigris.org/issues/show_bug.cgi?id=3020#desc16
  # svnadmin_tests.py 20 'don't filter mergeinfo revs from incremental dump'
  # also covers this problem.  Same problem here---
  #                                                |
  #   Properties on 'branches\B1\B\E':             |
  #     svn:mergeinfo                              |
  #       /branches/B2/B/E:6-7                     |
  #       /trunk/B/E:3-4,4-2 <---------------------
  #   Properties on 'branches\B2':
  #     svn:mergeinfo
  #       /trunk:4
  #
  # This test is set as XFail until --skip-missing-merge-sources gains its
  # new filtering functionality.
  url = sbox.repo_url + "/branches"
  expected_output = svntest.verify.UnorderedOutput([
    url + "/B1 - /branches/B2:6-7\n",
    "/trunk:1,4\n",
    url + "/B2 - /trunk:4\n",
    url + "/B1/B/E - /branches/B2/B/E:6-7\n",
    "/trunk/B/E:1,3-4\n"])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              reflect_dropped_renumbered_revs,
              svndumpfilter_loses_mergeinfo,
              dumpfilter_with_targets,
              dumpfilter_with_patterns,
              XFail(filter_mergeinfo_revs_outside_of_dump_stream),
              ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
<commit_msg>Let the svndumpfilter_tests.py tests see the stderr results when running svndumpfilter in non-quiet mode.<commit_after>#!/usr/bin/env python
#
#  svndumpfilter_tests.py:  testing the 'svndumpfilter' tool.
#
#  Subversion is a tool for revision control.
#  See http://subversion.apache.org for more information.
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
######################################################################

# General modules
import os
import sys
import tempfile

# Our testing module
import svntest
from svntest.verify import SVNExpectedStdout, SVNExpectedStderr

# Get some helper routines from svnadmin_tests
from svnadmin_tests import load_and_verify_dumpstream, test_create

# (abbreviation)
Skip = svntest.testcase.Skip
SkipUnless = svntest.testcase.SkipUnless
XFail = svntest.testcase.XFail
Item = svntest.wc.StateItem

######################################################################
# Helper routines

def filter_and_return_output(dump, bufsize=0, *varargs):
  """Filter the array of lines passed in 'dump' and return the output
  and errput"""

  if isinstance(dump, str):
    dump = [ dump ]

  # Does the caller want the stderr?
  try:
      varargs.index('-q')
      expected_errput = None # Stderr with -q or --quiet is a real error!
  except:
      try:
          varargs.index('--quiet')
          expected_errput = None
      except:
          expected_errput = svntest.verify.AnyOutput
  ## TODO: Should we handle exit_code?
  exit_code, output, errput = svntest.main.run_command_stdin(
    svntest.main.svndumpfilter_binary, expected_errput, bufsize, 1,
    dump, *varargs)

  return output, errput

######################################################################
# Tests

def reflect_dropped_renumbered_revs(sbox):
  "reflect dropped renumbered revs in svn:mergeinfo"

  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=2982. ##

  # Test svndumpfilter with include option
  test_create(sbox)
  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'with_merges.dump')
  dumpfile = open(dumpfile_location).read()

  filtered_out, filtered_err = filter_and_return_output(
      dumpfile, 0, "include",
      "trunk", "branch1",
      "--skip-missing-merge-sources",
      "--drop-empty-revs",
      "--renumber-revs", "--quiet")

  load_and_verify_dumpstream(sbox, [], [], None, filtered_out,
                             "--ignore-uuid")

  # Verify the svn:mergeinfo properties
  url = sbox.repo_url
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - /branch1:4-5\n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

  # Test svndumpfilter with exclude option
  test_create(sbox)
  filtered_out, filtered_err = filter_and_return_output(
      dumpfile, 0, "exclude", "branch1",
      "--skip-missing-merge-sources",
      "--drop-empty-revs",
      "--renumber-revs", "--quiet")

  load_and_verify_dumpstream(sbox, [], [], None, filtered_out,
                             "--ignore-uuid")

  # Verify the svn:mergeinfo properties
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - \n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

def svndumpfilter_loses_mergeinfo(sbox):
  "svndumpfilter loses mergeinfo"
  #svndumpfilter loses mergeinfo if invoked without --renumber-revs

  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=3181. ##

  test_create(sbox)
  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'with_merges.dump')
  dumpfile = open(dumpfile_location).read()

  filtered_out, filtered_err = filter_and_return_output(dumpfile, 0, "include",
                                                        "trunk", "branch1",
                                                        "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_out)

  # Verify the svn:mergeinfo properties
  url = sbox.repo_url
  expected_output = svntest.verify.UnorderedOutput([
    url + "/trunk - /branch1:4-8\n",
    ])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

def _simple_dumpfilter_test(sbox, dumpfile, *dumpargs):
  wc_dir = sbox.wc_dir

  filtered_output, filtered_err = filter_and_return_output(dumpfile, 0,
                                                           '--quiet',
                                                           *dumpargs)

  # Setup our expectations
  load_and_verify_dumpstream(sbox, [], [], None, filtered_output,
                             '--ignore-uuid')
  expected_disk = svntest.main.greek_state.copy()
  expected_disk.remove('A/B/E/alpha')
  expected_disk.remove('A/B/E/beta')
  expected_disk.remove('A/B/E')
  expected_disk.remove('A/D/H/chi')
  expected_disk.remove('A/D/H/psi')
  expected_disk.remove('A/D/H/omega')
  expected_disk.remove('A/D/H')
  expected_disk.remove('A/D/G/pi')
  expected_disk.remove('A/D/G/rho')
  expected_disk.remove('A/D/G/tau')
  expected_disk.remove('A/D/G')

  expected_output = svntest.wc.State(wc_dir, {
    'A'           : Item(status='A '),
    'A/B'         : Item(status='A '),
    'A/B/lambda'  : Item(status='A '),
    'A/B/F'       : Item(status='A '),
    'A/mu'        : Item(status='A '),
    'A/C'         : Item(status='A '),
    'A/D'         : Item(status='A '),
    'A/D/gamma'   : Item(status='A '),
    'iota'        : Item(status='A '),
    })

  expected_status = svntest.actions.get_virginal_state(wc_dir, 1)
  expected_status.remove('A/B/E/alpha')
  expected_status.remove('A/B/E/beta')
  expected_status.remove('A/B/E')
  expected_status.remove('A/D/H/chi')
  expected_status.remove('A/D/H/psi')
  expected_status.remove('A/D/H/omega')
  expected_status.remove('A/D/H')
  expected_status.remove('A/D/G/pi')
  expected_status.remove('A/D/G/rho')
  expected_status.remove('A/D/G/tau')
  expected_status.remove('A/D/G')

  # Check that our paths really were excluded
  svntest.actions.run_and_verify_update(wc_dir,
                                        expected_output,
                                        expected_disk,
                                        expected_status)

def dumpfilter_with_targets(sbox):
  "svndumpfilter --targets blah"
  ## See http://subversion.tigris.org/issues/show_bug.cgi?id=2697. ##

  test_create(sbox)

  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'greek_tree.dump')
  dumpfile = open(dumpfile_location).read()

  (fd, targets_file) = tempfile.mkstemp(dir=svntest.main.temp_dir)
  try:
    targets = open(targets_file, 'w')
    targets.write('/A/D/H\n')
    targets.write('/A/D/G\n')
    targets.close()
    _simple_dumpfilter_test(sbox, dumpfile,
                            'exclude', '/A/B/E', '--targets', targets_file)
  finally:
    os.close(fd)
    os.remove(targets_file)

def dumpfilter_with_patterns(sbox):
  "svndumpfilter --pattern PATH_PREFIX"

  test_create(sbox)

  dumpfile_location = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'greek_tree.dump')
  dumpfile = open(dumpfile_location).read()
  _simple_dumpfilter_test(sbox, dumpfile,
                          'exclude', '--pattern', '/A/D/[GH]*', '/A/[B]/E*')

#----------------------------------------------------------------------
# More testing for issue #3020 'Reflect dropped/renumbered revisions in
# svn:mergeinfo data during svnadmin load'
#
# Specifically, test that svndumpfilter, when used with the
# --skip-missing-merge-sources option, removes mergeinfo that refers to
# revisions that are older than the oldest revision in the dump stream.
def filter_mergeinfo_revs_outside_of_dump_stream(sbox):
  "filter mergeinfo revs outside of dump stream"

  test_create(sbox)

  # Load a partial dump into an existing repository.
  #
  # Picture == 1k words:
  #
  # The dump file we filter in this test, 'mergeinfo_included_partial.dump', is
  # a dump of r6:HEAD of the following repos:
  #
  #                       __________________________________________
  #                      |                                         |
  #                      |             ____________________________|_____
  #                      |            |                            |     |
  # trunk---r2---r3-----r5---r6-------r8---r9--------------->      |     |
  #   r1             |        |     |       |                      |     |
  # intial           |        |     |       |______                |     |
  # import         copy       |   copy             |            merge   merge
  #                  |        |     |            merge           (r5)   (r8)
  #                  |        |     |            (r9)              |     |
  #                  |        |     |              |               |     |
  #                  |        |     V              V               |     |
  #                  |        | branches/B2-------r11---r12---->   |     |
  #                  |        |     r7              |____|         |     |
  #                  |        |                        |           |     |
  #                  |      merge                      |___        |     |
  #                  |      (r6)                           |       |     |
  #                  |        |_________________           |       |     |
  #                  |                          |        merge     |     |
  #                  |                          |      (r11-12)    |     |
  #                  |                          |          |       |     |
  #                  V                          V          V       |     |
  #              branches/B1-------------------r10--------r13-->   |     |
  #                  r4                                            |     |
  #                   |                                            V     V
  #                  branches/B1/B/E------------------------------r14---r15->
  #
  #
  # The mergeinfo on the complete repos would look like this:
  #
  #   Properties on 'branches/B1':
  #     svn:mergeinfo
  #       /branches/B2:11-12
  #       /trunk:6,9
  #   Properties on 'branches/B1/B/E':
  #     svn:mergeinfo
  #       /branches/B2/B/E:11-12
  #       /trunk/B/E:5-6,8-9
  #   Properties on 'branches/B2':
  #     svn:mergeinfo
  #       /trunk:9
  #
  # We will run the partial dump through svndumpfilter using the the
  # --skip-missing-merge-soruces which should strip out any revisions < 6.
  # Then we'll load the filtered result into an empty repository.  This
  # should offset the incoming mergeinfo by -5.  The resulting mergeinfo
  # should look like this:
  #
  #   Properties on 'branches/B1':
  #     svn:mergeinfo
  #       /branches/B2:6-7
  #       /trunk:1,4
  #   Properties on 'branches/B1/B/E':
  #     svn:mergeinfo
  #       /branches/B2/B/E:6-7
  #       /trunk/B/E:1,3-4
  #   Properties on 'branches/B2':
  #     svn:mergeinfo
  #       /trunk:4
  partial_dump = os.path.join(os.path.dirname(sys.argv[0]),
                                   'svndumpfilter_tests_data',
                                   'mergeinfo_included_partial.dump')
  partial_dump_contents = open(partial_dump).read()
  filtered_dumpfile2, filtered_out = filter_and_return_output(
      partial_dump_contents,
      8192, # Set a sufficiently large bufsize to avoid a deadlock
      "include", "trunk", "branches",
      "--skip-missing-merge-sources",
      "--quiet")
  load_and_verify_dumpstream(sbox, [], [], None, filtered_dumpfile2,
                             '--ignore-uuid')
  # Check the resulting mergeinfo.
  #
  # Currently this fails with this mergeinfo:
  #
  #   Properties on 'branches\B1':
  #     svn:mergeinfo
  #       /branches/B2:6-7
  #       /trunk:4,5-2
  #                ^^^
  # What's this?!?!  It's corrupt mergeinfo caused when svnadmin load
  # maps the end rev of the incoming 5-6, but not the start rev, see '2)'
  # in http://subversion.tigris.org/issues/show_bug.cgi?id=3020#desc16
  # svnadmin_tests.py 20 'don't filter mergeinfo revs from incremental dump'
  # also covers this problem.  Same problem here---
  #                                                |
  #   Properties on 'branches\B1\B\E':             |
  #     svn:mergeinfo                              |
  #       /branches/B2/B/E:6-7                     |
  #       /trunk/B/E:3-4,4-2 <---------------------
  #   Properties on 'branches\B2':
  #     svn:mergeinfo
  #       /trunk:4
  #
  # This test is set as XFail until --skip-missing-merge-sources gains its
  # new filtering functionality.
  url = sbox.repo_url + "/branches"
  expected_output = svntest.verify.UnorderedOutput([
    url + "/B1 - /branches/B2:6-7\n",
    "/trunk:1,4\n",
    url + "/B2 - /trunk:4\n",
    url + "/B1/B/E - /branches/B2/B/E:6-7\n",
    "/trunk/B/E:1,3-4\n"])
  svntest.actions.run_and_verify_svn(None, expected_output, [],
                                     'propget', 'svn:mergeinfo', '-R',
                                     sbox.repo_url)

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              reflect_dropped_renumbered_revs,
              svndumpfilter_loses_mergeinfo,
              dumpfilter_with_targets,
              dumpfilter_with_patterns,
              XFail(filter_mergeinfo_revs_outside_of_dump_stream),
              ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import subprocess

from swid_generator.command_manager import CommandManager
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class DpkgEnvironment(CommonEnvironment):
    """
    Environment class for distributions using dpkg as package manager (e.g.
    Ubuntu, Debian or Mint).

    The packages are retrieved from the database using the ``dpkg-query`` tool:
    http://man7.org/linux/man-pages/man1/dpkg-query.1.html

    """
    executable_query = 'dpkg-query'
    executable = 'dpkg'
    md5_hash_length = 32
    CONFFILE_FILE_NAME = 'conffiles'
    CONTROL_ARCHIVE = 'control.tar.gz'

    installed_states = {
        'install ok installed': True,
        'deinstall ok config-files': False
    }

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """
        result = []
        command_args = [cls.executable_query,
                        '-W', '-f=${Package}\\n${Version}\\n${Status}\\n${conffiles}\\t']

        command_output = CommandManager.run_command_check_output(command_args)

        line_list = command_output.split('\t')

        for line in line_list:
            split_line = line.split('\n')

            if len(split_line) >= 4:
                package_info = PackageInfo()
                package_info.package = split_line[0]
                package_info.version = split_line[1]
                package_info.status = split_line[2]

                result.append(package_info)

        return [r for r in result if cls._package_installed(r)]

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Args:
            package_info (PackageInfo):
                The ``PackageInfo`` instance for the query.

        Returns:
            List of ``FileInfo`` instances.

        """
        result = []

        command_args_normal_files = [cls.executable_query, '-L', package_info.package]
        command_normal_file_output = CommandManager.run_command_check_output(command_args_normal_files)

        lines = command_normal_file_output.rstrip().split('\n')
        normal_files = filter(cls._is_file, lines)

        command_args_config_files = [cls.executable_query, '-W', '-f=${conffiles}\\n', package_info.package]
        command_config_file_output = CommandManager.run_command_check_output(command_args_config_files)

        lines = command_config_file_output.split('\n')
        stripped_lines = []

        for line in lines:
            if len(line) != 0:
                path_without_md5 = line.strip()[:len(line.strip()) - cls.md5_hash_length].strip()
                stripped_lines.append(path_without_md5)

        config_files = filter(cls._is_file, stripped_lines)

        for config_file_path in config_files:
            file_info = FileInfo(config_file_path)
            file_info.mutable = True
            result.append(file_info)

        for file_path in normal_files:
            if file_path not in config_files:
                result.append(FileInfo(file_path))

        return result

    @classmethod
    def _package_installed(cls, package_info):
        """
        Try to find out whether the specified package is installed or not.

        If the installed state cannot be determined with certainty we assume
        it's installed.

        Args:
            package_info (package_info.PackageInfo):
                The package to check.

        Returns:
            True or False

        """
        return cls.installed_states.get(package_info.status, True)

    @classmethod
    def get_files_from_packagefile(cls, file_pathname):
        """
        Extract all information of a .deb package.
        - List of all files
        - List of all Configuration-files

        This Method extract all the information in a temporary directory. The Debian package
        is extracted to the temporary directory, this because the files are needed to compute the File-Hash.

        :param file_pathname: Path to the .deb package
        :return: Lexicographical sorted List of FileInfo()-Objects (Conffiles and normal Files)
        """
        save_options = cls._create_temp_folder(file_pathname)
        result = []

        command_args_unpack_package = [cls.executable, '-x', save_options['absolute_package_path'],
                                       save_options['save_location']]

        command_args_extract_controlpackage = ["ar", "x", save_options['absolute_package_path'],
                                               cls.CONTROL_ARCHIVE]

        command_args_extract_conffile = ["tar", "-zxf", "/".join((save_options['save_location'],
                                                                  cls.CONTROL_ARCHIVE)), "./conffiles"]

        CommandManager.run_command(command_args_unpack_package)

        try:
            CommandManager.run_command(command_args_extract_controlpackage,
                                       working_directory=save_options['save_location'])
            CommandManager.run_command(command_args_extract_conffile,
                                       working_directory=save_options['save_location'])

            conffile_save_location = "/".join((save_options['save_location'], cls.CONFFILE_FILE_NAME))

            with open(conffile_save_location, 'rb') as afile:
                file_content = afile.read().encode('utf-8')

            config_file_paths = filter(lambda path: len(path) > 0, file_content.split('\n'))

            for config_file_path in config_file_paths:
                file_info = FileInfo(config_file_path, actual_path=False)
                file_info.set_actual_path(save_options['save_location'] + config_file_path)
                file_info.mutable = True
                result.append(file_info)

        except(IOError, subprocess.CalledProcessError):
            config_file_paths = []

        command_args_file_list = [cls.executable, '-c', file_pathname]
        command_output_list = CommandManager.run_command_check_output(command_args_file_list)

        line_list = command_output_list.split('\n')

        for line in line_list:
            splitted_line_array = line.split(' ')

            if '->' in splitted_line_array:
                # symbol-link
                directory_or_file_path = splitted_line_array[-3]
            else:
                # Last-Entry from Array is File-Path
                directory_or_file_path = splitted_line_array[-1]

            path_without_leading_point = directory_or_file_path[1:]

            temp_save_location = str("/".join((save_options['save_location'], path_without_leading_point)))

            if cls._is_file(temp_save_location):
                if path_without_leading_point not in config_file_paths:
                    file_info = FileInfo(path_without_leading_point, actual_path=False)
                    file_info.set_actual_path(temp_save_location)
                    result.append(file_info)

        return sorted(result, key=lambda f: f.full_pathname)

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        """
        Extract the Package-Name and the Package-Version from the Debian-Package.

        :param file_path: Path to the Debian-Package
        :return: A PackageInfo()-Object with Package-Version and Package-Name.
        """
        command_args_packagename = [cls.executable, '-f', file_path, 'Package']
        command_args_version = [cls.executable, '-f', file_path, 'Version']
        package_name = CommandManager.run_command_check_output(command_args_packagename)
        package_version = CommandManager.run_command_check_output(command_args_version)

        package_info = PackageInfo()
        package_info.package = package_name.strip()
        package_info.version = package_version.strip()

        return package_info
<commit_msg>Symbol-Links in list extracted<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import subprocess
import ntpath

from swid_generator.command_manager import CommandManager
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class DpkgEnvironment(CommonEnvironment):
    """
    Environment class for distributions using dpkg as package manager (e.g.
    Ubuntu, Debian or Mint).

    The packages are retrieved from the database using the ``dpkg-query`` tool:
    http://man7.org/linux/man-pages/man1/dpkg-query.1.html

    """
    executable_query = 'dpkg-query'
    executable = 'dpkg'
    md5_hash_length = 32
    CONFFILE_FILE_NAME = 'conffiles'
    CONTROL_ARCHIVE = 'control.tar.gz'

    installed_states = {
        'install ok installed': True,
        'deinstall ok config-files': False
    }

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """
        result = []
        command_args = [cls.executable_query,
                        '-W', '-f=${Package}\\n${Version}\\n${Status}\\n${conffiles}\\t']

        command_output = CommandManager.run_command_check_output(command_args)

        line_list = command_output.split('\t')

        for line in line_list:
            split_line = line.split('\n')

            if len(split_line) >= 4:
                package_info = PackageInfo()
                package_info.package = split_line[0]
                package_info.version = split_line[1]
                package_info.status = split_line[2]

                result.append(package_info)

        return [r for r in result if cls._package_installed(r)]

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Args:
            package_info (PackageInfo):
                The ``PackageInfo`` instance for the query.

        Returns:
            List of ``FileInfo`` instances.

        """
        result = []

        command_args_normal_files = [cls.executable_query, '-L', package_info.package]
        command_normal_file_output = CommandManager.run_command_check_output(command_args_normal_files)

        lines = command_normal_file_output.rstrip().split('\n')
        normal_files = filter(cls._is_file, lines)

        command_args_config_files = [cls.executable_query, '-W', '-f=${conffiles}\\n', package_info.package]
        command_config_file_output = CommandManager.run_command_check_output(command_args_config_files)

        lines = command_config_file_output.split('\n')
        stripped_lines = []

        for line in lines:
            if len(line) != 0:
                path_without_md5 = line.strip()[:len(line.strip()) - cls.md5_hash_length].strip()
                stripped_lines.append(path_without_md5)

        config_files = filter(cls._is_file, stripped_lines)

        for config_file_path in config_files:
            file_info = FileInfo(config_file_path)
            file_info.mutable = True
            result.append(file_info)

        for file_path in normal_files:
            if file_path not in config_files:
                result.append(FileInfo(file_path))

        return result

    @classmethod
    def _package_installed(cls, package_info):
        """
        Try to find out whether the specified package is installed or not.

        If the installed state cannot be determined with certainty we assume
        it's installed.

        Args:
            package_info (package_info.PackageInfo):
                The package to check.

        Returns:
            True or False

        """
        return cls.installed_states.get(package_info.status, True)

    @classmethod
    def get_files_from_packagefile(cls, file_pathname):
        """
        Extract all information of a .deb package.
        - List of all files
        - List of all Configuration-files

        This Method extract all the information in a temporary directory. The Debian package
        is extracted to the temporary directory, this because the files are needed to compute the File-Hash.

        :param file_pathname: Path to the .deb package
        :return: Lexicographical sorted List of FileInfo()-Objects (Conffiles and normal Files)
        """
        save_options = cls._create_temp_folder(file_pathname)
        result = []
        result_help_list = []  # needed to check duplications

        command_args_unpack_package = [cls.executable, '-x', save_options['absolute_package_path'],
                                       save_options['save_location']]

        command_args_extract_controlpackage = ["ar", "x", save_options['absolute_package_path'],
                                               cls.CONTROL_ARCHIVE]

        command_args_extract_conffile = ["tar", "-zxf", "/".join((save_options['save_location'],
                                                                  cls.CONTROL_ARCHIVE)), "./conffiles"]

        CommandManager.run_command(command_args_unpack_package)

        try:
            CommandManager.run_command(command_args_extract_controlpackage,
                                       working_directory=save_options['save_location'])
            CommandManager.run_command(command_args_extract_conffile,
                                       working_directory=save_options['save_location'])

            conffile_save_location = "/".join((save_options['save_location'], cls.CONFFILE_FILE_NAME))

            with open(conffile_save_location, 'rb') as afile:
                file_content = afile.read().encode('utf-8')

            config_file_paths = filter(lambda path: len(path) > 0, file_content.split('\n'))

            for config_file_path in config_file_paths:
                file_info = FileInfo(config_file_path, actual_path=False)
                file_info.set_actual_path(save_options['save_location'] + config_file_path)
                file_info.mutable = True
                result.append(file_info)

        except(IOError, subprocess.CalledProcessError):
            config_file_paths = []

        command_args_file_list = [cls.executable, '-c', file_pathname]
        command_output_list = CommandManager.run_command_check_output(command_args_file_list)

        line_list = command_output_list.split('\n')

        for line in line_list:
            splitted_line_array = line.split(' ')
            print(splitted_line_array)
            if '->' in splitted_line_array:
                # symbol-link
                directory_or_file_path = splitted_line_array[-1]
                symbol_link = splitted_line_array[-3]
                head, file_name = ntpath.split(symbol_link)

                if "../" in directory_or_file_path:
                    root, folder_name = ntpath.split(head)
                    directory_or_file_path = root + directory_or_file_path[2:]
                else:
                    directory_or_file_path = "/".join((head, directory_or_file_path))

            else:
                # Last-Entry from Array is File-Path
                directory_or_file_path = splitted_line_array[-1]

            path_without_leading_point = directory_or_file_path[1:]

            temp_save_location = str("/".join((save_options['save_location'], path_without_leading_point)))

            if cls._is_file(temp_save_location):
                if path_without_leading_point not in config_file_paths and path_without_leading_point not in result_help_list:
                    result_help_list.append(path_without_leading_point)
                    file_info = FileInfo(path_without_leading_point, actual_path=False)
                    file_info.set_actual_path(temp_save_location)
                    result.append(file_info)

        return sorted(result, key=lambda f: f.full_pathname)

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        """
        Extract the Package-Name and the Package-Version from the Debian-Package.

        :param file_path: Path to the Debian-Package
        :return: A PackageInfo()-Object with Package-Version and Package-Name.
        """
        command_args_packagename = [cls.executable, '-f', file_path, 'Package']
        command_args_version = [cls.executable, '-f', file_path, 'Version']
        package_name = CommandManager.run_command_check_output(command_args_packagename)
        package_version = CommandManager.run_command_check_output(command_args_version)

        package_info = PackageInfo()
        package_info.package = package_name.strip()
        package_info.version = package_version.strip()

        return package_info
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import subprocess
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class DpkgEnvironment(CommonEnvironment):
    """
    Environment class for distributions using dpkg as package manager (e.g.
    Ubuntu, Debian or Mint).

    The packages are retrieved from the database using the ``dpkg-query`` tool:
    http://man7.org/linux/man-pages/man1/dpkg-query.1.html

    """
    executable = 'dpkg-query'
    executable_dpkg = 'dpkg'
    CONFFILE_FILE_NAME = 'conffiles'
    CONTROL_ARCHIVE = 'control.tar.gz'

    installed_states = {
        'install ok installed': True,
        'deinstall ok config-files': False
    }

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """
        command_args = [cls.executable, '-W', '-f=${Package}\\n${Version}\\n${Status}\\n${conffiles}\\t']
        data = subprocess.check_output(command_args)
        result = []

        if isinstance(data, bytes):
            data = data.decode('utf-8')
        line_list = data.split('\t')

        for line in line_list:
            split_line = line.split('\n')

            if len(split_line) >= 4:
                package_info = PackageInfo()
                package_info.package = split_line[0]
                package_info.version = split_line[1]
                package_info.status = split_line[2]

                # if Config-Files exists
                if split_line[3] != '':
                    config_files = []
                    for file_path in split_line[3:len(split_line)]:
                        file_path_without_md5 = file_path.split(' ')[1]
                        if cls._is_file(file_path_without_md5):
                            file_info = FileInfo(file_path_without_md5)
                            file_info.mutable = True
                            config_files.append(file_info)

                    package_info.files.extend(config_files)

                result.append(package_info)

        return [r for r in result if cls._package_installed(r)]

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Args:
            package_name (str):
                The package name as string (e.g. ``cowsay``).

        Returns:
            List of ``FileInfo`` instances.

        """
        command_args = [cls.executable, '-L', package_info.package]
        data = subprocess.check_output(command_args)
        if isinstance(data, bytes):
            data = data.decode('utf-8')
        lines = data.rstrip().split('\n')
        files = filter(cls._is_file, lines)
        result_list = []

        for path in files:
            if not any(file_info.full_pathname.strip() == path for file_info in package_info.files):
                result_list.append(FileInfo(path))
        return result_list

    @classmethod
    def _package_installed(cls, package_info):
        """
        Try to find out whether the specified package is installed or not.

        If the installed state cannot be determined with certainty we assume
        it's installed.

        Args:
            package_info (package_info.PackageInfo):
                The package to check.

        Returns:
            True or False

        """
        return cls.installed_states.get(package_info.status, True)

    @classmethod
    def get_files_from_packagefile(cls, file_fullpathname):
        """
        Extract all information of a .deb package.
        - List of all files
        - List of all Configuration-files

        This Method extract all the information in a temporary directory. The Debian package
        is extracted to the temporary directory, this because the files are needed to compute the File-Hash.

        :param file_fullpathname: Path to the .deb package
        :return: Lexicographical sorted List of FileInfo()-Objects (Conffiles and normal Files)
        """
        save_options = cls._create_temp_folder(file_fullpathname)
        all_files = []

        # unpack .deb-Packagefile
        subprocess.call([cls.executable_dpkg, '-x', file_fullpathname, save_options['save_location']])

        # add Config-Files
        subprocess.call(["ar", "x", save_options['absolute_package_path'], cls.CONTROL_ARCHIVE])
        subprocess.call(["mv", cls.CONTROL_ARCHIVE, save_options['save_location']])
        subprocess.call(["tar", "-zxf",
                         "/".join((save_options['save_location'], cls.CONTROL_ARCHIVE)), "./conffiles"])
        subprocess.call(["mv", cls.CONFFILE_FILE_NAME, save_options['save_location']])

        conffile_save_location = "/".join((save_options['save_location'], cls.CONFFILE_FILE_NAME))

        with open(conffile_save_location, 'rb') as afile:
            file_content = afile.read().encode('utf-8')

        config_file_paths = filter(lambda path: len(path) > 0, file_content.split('\n'))

        for config_file_path in config_file_paths:
            file_info = FileInfo(config_file_path, actual_path=False)
            file_info.set_actual_path(save_options['save_location'] + config_file_path)
            file_info.mutable = True
            all_files.append(file_info)

        # extract File-List from Package
        command_args = [cls.executable_dpkg, '-c', file_fullpathname]
        data = subprocess.check_output(command_args)

        if isinstance(data, bytes):
            data = data.decode('utf-8')

        line_list = data.split('\n')

        for line in line_list:
            splitted_line_array = line.split(' ')

            # Last-Entry from Array is File-Path
            directory_or_file_path = splitted_line_array[-1]
            path_without_leading_point = directory_or_file_path[1:len(directory_or_file_path)]

            temp_save_location = str("/".join((save_options['save_location'], path_without_leading_point)))

            if cls._is_file(temp_save_location):
                if path_without_leading_point not in config_file_paths:
                    file_info = FileInfo(path_without_leading_point, actual_path=False)
                    file_info.set_actual_path(save_options['save_location'] + path_without_leading_point)
                    all_files.append(file_info)

        return sorted(all_files, key=lambda f: f.full_pathname)

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        """
        Extract the Package-Name and the Package-Version from the Debian-Package.

        :param file_path: Path to the Debian-Package
        :return: A PackageInfo()-Object with Package-Version and Package-Name.
        """
        command_args = ['dpkg', '-f', file_path, 'Package']
        package_name = subprocess.check_output(command_args)
        if isinstance(package_name, bytes):
            package_name = package_name.decode('utf-8')
        # extract Package-Version
        command_args = ['dpkg', '-f', file_path, 'Version']
        package_version = subprocess.check_output(command_args)
        if isinstance(package_version, bytes):
            package_version = package_version.decode('utf-8')

        package_info = PackageInfo()
        package_info.package = package_name.strip()
        package_info.version = package_version.strip()

        return package_info
<commit_msg>Error-Handling on conffiles extraction added; If file not extracted, no configuration-files exists<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import subprocess
import os
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class DpkgEnvironment(CommonEnvironment):
    """
    Environment class for distributions using dpkg as package manager (e.g.
    Ubuntu, Debian or Mint).

    The packages are retrieved from the database using the ``dpkg-query`` tool:
    http://man7.org/linux/man-pages/man1/dpkg-query.1.html

    """
    executable = 'dpkg-query'
    executable_dpkg = 'dpkg'
    CONFFILE_FILE_NAME = 'conffiles'
    CONTROL_ARCHIVE = 'control.tar.gz'

    installed_states = {
        'install ok installed': True,
        'deinstall ok config-files': False
    }

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """
        command_args = [cls.executable, '-W', '-f=${Package}\\n${Version}\\n${Status}\\n${conffiles}\\t']
        data = subprocess.check_output(command_args)
        result = []

        if isinstance(data, bytes):
            data = data.decode('utf-8')
        line_list = data.split('\t')

        for line in line_list:
            split_line = line.split('\n')

            if len(split_line) >= 4:
                package_info = PackageInfo()
                package_info.package = split_line[0]
                package_info.version = split_line[1]
                package_info.status = split_line[2]

                # if Config-Files exists
                if split_line[3] != '':
                    config_files = []
                    for file_path in split_line[3:len(split_line)]:
                        file_path_without_md5 = file_path.split(' ')[1]
                        if cls._is_file(file_path_without_md5):
                            file_info = FileInfo(file_path_without_md5)
                            file_info.mutable = True
                            config_files.append(file_info)

                    package_info.files.extend(config_files)

                result.append(package_info)

        return [r for r in result if cls._package_installed(r)]

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Args:
            package_name (str):
                The package name as string (e.g. ``cowsay``).

        Returns:
            List of ``FileInfo`` instances.

        """
        command_args = [cls.executable, '-L', package_info.package]
        data = subprocess.check_output(command_args)
        if isinstance(data, bytes):
            data = data.decode('utf-8')
        lines = data.rstrip().split('\n')
        files = filter(cls._is_file, lines)
        result_list = []

        for path in files:
            if not any(file_info.full_pathname.strip() == path for file_info in package_info.files):
                result_list.append(FileInfo(path))
        return result_list

    @classmethod
    def _package_installed(cls, package_info):
        """
        Try to find out whether the specified package is installed or not.

        If the installed state cannot be determined with certainty we assume
        it's installed.

        Args:
            package_info (package_info.PackageInfo):
                The package to check.

        Returns:
            True or False

        """
        return cls.installed_states.get(package_info.status, True)

    @classmethod
    def get_files_from_packagefile(cls, file_fullpathname):
        """
        Extract all information of a .deb package.
        - List of all files
        - List of all Configuration-files

        This Method extract all the information in a temporary directory. The Debian package
        is extracted to the temporary directory, this because the files are needed to compute the File-Hash.

        :param file_fullpathname: Path to the .deb package
        :return: Lexicographical sorted List of FileInfo()-Objects (Conffiles and normal Files)
        """
        save_options = cls._create_temp_folder(file_fullpathname)
        all_files = []

        # unpack .deb-Packagefile
        subprocess.call([cls.executable_dpkg, '-x', file_fullpathname, save_options['save_location']])

        try:
            # add Config-Files
            with open(os.devnull, 'w') as devnull:
                subprocess.call(["ar", "x", save_options['absolute_package_path'], cls.CONTROL_ARCHIVE],
                                stderr=devnull)
                subprocess.call(["mv", cls.CONTROL_ARCHIVE, save_options['save_location']], stderr=devnull)
                subprocess.call(["tar", "-zxf",
                                 "/".join((save_options['save_location'], cls.CONTROL_ARCHIVE)),
                                 "./conffiles"], stderr=devnull)
                subprocess.call(["mv", cls.CONFFILE_FILE_NAME, save_options['save_location']],
                                stderr=devnull)

            conffile_save_location = "/".join((save_options['save_location'], cls.CONFFILE_FILE_NAME))

            with open(conffile_save_location, 'rb') as afile:
                file_content = afile.read().encode('utf-8')

            config_file_paths = filter(lambda path: len(path) > 0, file_content.split('\n'))

            for config_file_path in config_file_paths:
                file_info = FileInfo(config_file_path, actual_path=False)
                file_info.set_actual_path(save_options['save_location'] + config_file_path)
                file_info.mutable = True
                all_files.append(file_info)
        except(IOError, subprocess.CalledProcessError):
            config_file_paths = []

        # extract File-List from Package
        command_args = [cls.executable_dpkg, '-c', file_fullpathname]
        data = subprocess.check_output(command_args)

        if isinstance(data, bytes):
            data = data.decode('utf-8')

        line_list = data.split('\n')

        for line in line_list:
            splitted_line_array = line.split(' ')

            # Last-Entry from Array is File-Path
            directory_or_file_path = splitted_line_array[-1]
            path_without_leading_point = directory_or_file_path[1:len(directory_or_file_path)]

            temp_save_location = str("/".join((save_options['save_location'], path_without_leading_point)))

            if cls._is_file(temp_save_location):
                if path_without_leading_point not in config_file_paths:
                    file_info = FileInfo(path_without_leading_point, actual_path=False)
                    file_info.set_actual_path(temp_save_location)
                    all_files.append(file_info)

        return sorted(all_files, key=lambda f: f.full_pathname)

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        """
        Extract the Package-Name and the Package-Version from the Debian-Package.

        :param file_path: Path to the Debian-Package
        :return: A PackageInfo()-Object with Package-Version and Package-Name.
        """
        command_args = ['dpkg', '-f', file_path, 'Package']
        package_name = subprocess.check_output(command_args)
        if isinstance(package_name, bytes):
            package_name = package_name.decode('utf-8')
        # extract Package-Version
        command_args = ['dpkg', '-f', file_path, 'Version']
        package_version = subprocess.check_output(command_args)
        if isinstance(package_version, bytes):
            package_version = package_version.decode('utf-8')

        package_info = PackageInfo()
        package_info.package = package_name.strip()
        package_info.version = package_version.strip()

        return package_info
<|endoftext|>