<commit_before>import enum
import sys
from typing import List, Tuple
from datetime import timedelta
from urllib.parse import urlparse

MSP_FILE_PLANS = [
    (4, 'STORAGE_100GB', '100GB'),
    (7, 'STORAGE_1000GB', '1TB'),
    (8, 'STORAGE_10000GB', '10TB'),
]

MSP_PLANS = [
    (1, 'business', 'Business', 4),
    (2, 'businessPlus', 'Business Plus', 7),
    (10, 'enterprise', 'Enterprise', 4),
    (11, 'enterprisePlus', 'Enterprise Plus', 7),
]

MSP_ADDONS = [
    ('enterprise_breach_watch', 'BreachWatch', False),
    ('compliance_report', 'Compliance Reporting', False),
    ('enterprise_audit_and_reporting', 'Advanced Reporting & Alerts Module', False),
    ('msp_service_and_support', 'MSP Dedicated Service & Support', False),
    ('secrets_manager', 'Keeper Secrets Manager (KSM)', False),
    ('connection_manager', 'Keeper Connection Manager (KCM)', True),
    ('chat', 'KeeperChat', False),
]

class PrivilegeScope(enum.IntEnum):
    All = 1,
    MSP = 2,
    Hidden = 3,

# Managed Role privileges

ROLE_PRIVILEGES = [
    ('Manage Nodes', 'MANAGE_NODES', PrivilegeScope.All),
    ('Manage Users', 'MANAGE_USER', PrivilegeScope.All),
    ('Manage Licences', 'MANAGE_LICENCES', PrivilegeScope.Hidden),
    ('Manage Roles', 'MANAGE_ROLES', PrivilegeScope.All),
    ('Manage Teams', 'MANAGE_TEAMS', PrivilegeScope.All),
    ('Run Security Reports', 'RUN_REPORTS', PrivilegeScope.All),
    ('Manage Bridge/SSO', 'MANAGE_BRIDGE', PrivilegeScope.All),
    ('Perform Device Approvals', 'APPROVE_DEVICE', PrivilegeScope.All),
    ('Manage Record Types in Vault', 'MANAGE_RECORD_TYPES', PrivilegeScope.All),
    ('Run Compliance Reports', 'RUN_COMPLIANCE_REPORTS', PrivilegeScope.All),
    ('Manage Companies', 'MANAGE_COMPANIES', PrivilegeScope.MSP),
    ('Transfer Account', 'TRANSFER_ACCOUNT', PrivilegeScope.All),
    ('Sharing Administrator', 'SHARING_ADMINISTRATOR', PrivilegeScope.All),
]

# Timeout constants
# Set to default value by using timedelta of 0
TIMEOUT_DEFAULT = timedelta(0)
TIMEOUT_MIN = timedelta(minutes=1)
TIMEOUT_DEFAULT_UNIT = 'minutes'
TIMEOUT_ALLOWED_UNITS = ('days', 'hours', 'minutes')

EMAIL_PATTERN = r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)"

# Enforcement constants

_ENFORCEMENT_GROUPS = [
    "LOGIN_SETTINGS",
    "TWO_FACTOR_AUTHENTICATION",
    "PLATFORM_RESTRICTION",
    "VAULT_FEATURES",
    "RECORD_TYPES",
    "SHARING_AND_UPLOADING",
    "KEEPER_FILL",
    "ACCOUNT_SETTINGS",
    "ALLOW_IP_LIST",
    "ACCOUNT_SETTINGS",
]

_ENFORCEMENTS = [
    ("MASTER_PASSWORD_MINIMUM_LENGTH", 10, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_SPECIAL", 11, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_UPPER", 12, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_LOWER", 13, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_DIGITS", 14, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_RESTRICT_DAYS_BEFORE_REUSE", 16, "LONG", "LOGIN_SETTINGS"),
    ("REQUIRE_TWO_FACTOR", 20, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("MASTER_PASSWORD_MAXIMUM_DAYS_BEFORE_CHANGE", 22, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_EXPIRED_AS_OF", 23, "LONG", "LOGIN_SETTINGS"),
    ("MINIMUM_PBKDF2_ITERATIONS", 55, "LONG", "ACCOUNT_SETTINGS"),
    ("MAX_SESSION_LOGIN_TIME", 24, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_PERSISTENT_LOGIN", 25, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("STAY_LOGGED_IN_DEFAULT", 26, "BOOLEAN", "ACCOUNT_ENFORCEMENTS"),
    ("RESTRICT_SHARING_ALL", 30, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_ENTERPRISE", 31, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_EXPORT", 32, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_IMPORT", 111, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_FILE_UPLOAD", 33, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("REQUIRE_ACCOUNT_SHARE", 34, "ACCOUNT_SHARE", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_INCOMING_ALL", 36, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_INCOMING_ENTERPRISE", 37, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTIRCT_SHARING_RECORD_AND_FOLDER", 120, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_RECORD_WITH_ATTACHMENTS", 121, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_IP_ADDRESSES", 40, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("REQUIRE_DEVICE_APPROVAL", 41, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("REQUIRE_ACCOUNT_RECOVERY_APPROVAL", 42, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_VAULT_IP_ADDRESSES", 43, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("TIP_ZONE_RESTRICT_ALLOWED_IP_RANGES", 44, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("AUTOMATIC_BACKUP_EVERY_X_DAYS", 45, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_OFFLINE_ACCESS", 46, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("SEND_INVITE_AT_REGISTRATION", 47, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_EMAIL_CHANGE", 48, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_IOS_FINGERPRINT", 49, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_MAC_FINGERPRINT", 50, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_ANDROID_FINGERPRINT", 51, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_WINDOWS_FINGERPRINT", 83, "BOOLEAN", "LOGIN_SETTINGS"),
    ("LOGOUT_TIMER_WEB", 52, "LONG", "ACCOUNT_SETTINGS"),
    ("LOGOUT_TIMER_MOBILE", 53, "LONG", "ACCOUNT_SETTINGS"),
    ("LOGOUT_TIMER_DESKTOP", 54, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_WEB_VAULT_ACCESS", 60, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_EXTENSIONS_ACCESS", 61, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_ACCESS", 62, " BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_ACCESS", 63, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_IOS_ACCESS", 64, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_ANDROID_ACCESS", 65, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_WINDOWS_PHONE_ACCESS", 66, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_WIN_ACCESS", 67, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_MAC_ACCESS", 68, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_CHAT_DESKTOP_ACCESS", 84, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_CHAT_MOBILE_ACCESS", 85, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_COMMANDER_ACCESS", 88, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_TEXT", 70, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_GOOGLE", 71, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_DNA", 72, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_DUO", 73, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_RSA", 74, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_WEB", 80, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_MOBILE", 81, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_DESKTOP", 82, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_SECURITY_KEYS", 86, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_BY_IP", 87, "JSONARRAY"),
    ("RESTRICT_DOMAIN_ACCESS", 90, "STRING", "KEEPER_FILL"),
    ("RESTRICT_DOMAIN_CREATE", 91, "STRING", "KEEPER_FILL"),
    ("RESTRICT_HOVER_LOCKS", 92, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_LOGIN", 93, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_FILL", 94, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_AUTO_SUBMIT", 95, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_SAVE", 96, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_CHANGE", 97, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_AUTO_FILL", 98, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_CREATE_FOLDER", 100, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_CREATE_IDENTITY_PAYMENT_RECORDS", 102, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_CUSTOM_FIELDS", 103, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_NOTES", 104, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_PASSWORDS_WHILE_EDITING", 105, "BOOLEAN", "VAULT_FEATURES"),
    ("GENERATED_PASSWORD_COMPLEXITY", 106, "STRING", "VAULT_FEATURES"),
    ("GENERATED_SECURITY_QUESTION_COMPLEXITY", 109, "STRING", "VAULT_FEATURES"),
    ("DAYS_BEFORE_DELETED_RECORDS_CLEARED_PERM", 107, "LONG", "VAULT_FEATURES"),
    ("DAYS_BEFORE_DELETED_RECORDS_AUTO_CLEARED", 108, "LONG", "VAULT_FEATURES"),
    ("ALLOW_ALTERNATE_PASSWORDS", 110, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_LINK_SHARING", 122, "BOOLEAN", "SHARING_ENFORCEMENTS"),
    ("DISABLE_SETUP_TOUR", 140, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_PERSONAL_LICENSE", 141, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("DISABLE_ONBOARDING", 142, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("DISALLOW_V2_CLIENTS", 143, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_IP_AUTOAPPROVAL", 144, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("SEND_BREACH_WATCH_EVENTS", 200, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_BREACH_WATCH", 201, "BOOLEAN", "VAULT_FEATURES"),
    ("RESEND_ENTERPRISE_INVITE_IN_X_DAYS", 202, "LONG", "ACCOUNT_SETTINGS"),
    ("MASTER_PASSWORD_REENTRY", 203, "JSON"),
    ("RESTRICT_ACCOUNT_RECOVERY", 204, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("KEEPER_FILL_HOVER_LOCKS", 205, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_AUTO_FILL", 206, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_AUTO_SUBMIT", 207, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_MATCH_ON_SUBDOMAIN", 208, "TERNARY_EDN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_DISABLE", 209, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_HTTP_FILL_WARNING", 210, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_RECORD_TYPES", 211, "RECORD_TYPES", "RECORD_TYPES"),
    ("ALLOW_SECRETS_MANAGER", 212, "BOOLEAN", "VAULT_FEATURES"),
    ("REQUIRE_SELF_DESTRUCT", 213, "BOOLEAN", "ACCOUNT_ENFORCEMENTS"),
]

def enforcement_list():  # type: () -> List[Tuple[str, str, str]]
    groups = {x[1]: x[0] for x in enumerate(_ENFORCEMENT_GROUPS)}
    enforcements = [(x[3], x[0], x[1], x[2]) for x in _ENFORCEMENTS if len(x) >= 4 and x[3]]
    enforcements.sort(key=lambda x: (groups[x[0]] if x[0] in groups else 100) * 1000 + x[2])
    return [(x[0].title().replace('_', ' '), x[1].lower(), x[3].lower()) for x in enforcements]

ENFORCEMENTS = {}
for e in _ENFORCEMENTS:
    ENFORCEMENTS[e[0].lower()] = e[2].lower()

# OS dependent constants
if sys.platform.startswith('win'):
    OS_WHICH_CMD = 'where'
else:
    OS_WHICH_CMD = 'which'

KEEPER_PUBLIC_HOSTS = {
    'US': 'keepersecurity.com',
    'EU': 'keepersecurity.eu',
    'AU': 'keepersecurity.com.au',
    'CA': 'keepersecurity.ca',
    'JP': 'keepersecurity.jp',
    'GOV': 'govcloud.keepersecurity.us'
}

def get_abbrev_by_host(host):
    # Return abbreviation of the Keeper's public host

    if host.startswith('https:'):
        host = urlparse(host).netloc    # https://keepersecurity.com/api/v2/ --> keepersecurity.com

    keys = [k for k, v in KEEPER_PUBLIC_HOSTS.items() if v == host]
    if keys:
        return keys[0]
    return None

# Messages
# Account Transfer
ACCOUNT_TRANSFER_MSG = """
Your Keeper administrator has enabled the ability to transfer your vault records
in accordance with company operating procedures and policies.
Please acknowledge this change in account settings by typing 'Accept'.
If you do not accept this change by {0}, you will be locked out of your account.
"""
<commit_msg>Enable or disable the 'Autofill suggestions'. KC-352<commit_after>import enum
import sys
from typing import List, Tuple
from datetime import timedelta
from urllib.parse import urlparse

MSP_FILE_PLANS = [
    (4, 'STORAGE_100GB', '100GB'),
    (7, 'STORAGE_1000GB', '1TB'),
    (8, 'STORAGE_10000GB', '10TB'),
]

MSP_PLANS = [
    (1, 'business', 'Business', 4),
    (2, 'businessPlus', 'Business Plus', 7),
    (10, 'enterprise', 'Enterprise', 4),
    (11, 'enterprisePlus', 'Enterprise Plus', 7),
]

MSP_ADDONS = [
    ('enterprise_breach_watch', 'BreachWatch', False),
    ('compliance_report', 'Compliance Reporting', False),
    ('enterprise_audit_and_reporting', 'Advanced Reporting & Alerts Module', False),
    ('msp_service_and_support', 'MSP Dedicated Service & Support', False),
    ('secrets_manager', 'Keeper Secrets Manager (KSM)', False),
    ('connection_manager', 'Keeper Connection Manager (KCM)', True),
    ('chat', 'KeeperChat', False),
]

class PrivilegeScope(enum.IntEnum):
    All = 1,
    MSP = 2,
    Hidden = 3,

# Managed Role privileges

ROLE_PRIVILEGES = [
    ('Manage Nodes', 'MANAGE_NODES', PrivilegeScope.All),
    ('Manage Users', 'MANAGE_USER', PrivilegeScope.All),
    ('Manage Licences', 'MANAGE_LICENCES', PrivilegeScope.Hidden),
    ('Manage Roles', 'MANAGE_ROLES', PrivilegeScope.All),
    ('Manage Teams', 'MANAGE_TEAMS', PrivilegeScope.All),
    ('Run Security Reports', 'RUN_REPORTS', PrivilegeScope.All),
    ('Manage Bridge/SSO', 'MANAGE_BRIDGE', PrivilegeScope.All),
    ('Perform Device Approvals', 'APPROVE_DEVICE', PrivilegeScope.All),
    ('Manage Record Types in Vault', 'MANAGE_RECORD_TYPES', PrivilegeScope.All),
    ('Run Compliance Reports', 'RUN_COMPLIANCE_REPORTS', PrivilegeScope.All),
    ('Manage Companies', 'MANAGE_COMPANIES', PrivilegeScope.MSP),
    ('Transfer Account', 'TRANSFER_ACCOUNT', PrivilegeScope.All),
    ('Sharing Administrator', 'SHARING_ADMINISTRATOR', PrivilegeScope.All),
]

# Timeout constants
# Set to default value by using timedelta of 0
TIMEOUT_DEFAULT = timedelta(0)
TIMEOUT_MIN = timedelta(minutes=1)
TIMEOUT_DEFAULT_UNIT = 'minutes'
TIMEOUT_ALLOWED_UNITS = ('days', 'hours', 'minutes')

EMAIL_PATTERN = r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)"

# Enforcement constants

_ENFORCEMENT_GROUPS = [
    "LOGIN_SETTINGS",
    "TWO_FACTOR_AUTHENTICATION",
    "PLATFORM_RESTRICTION",
    "VAULT_FEATURES",
    "RECORD_TYPES",
    "SHARING_AND_UPLOADING",
    "KEEPER_FILL",
    "ACCOUNT_SETTINGS",
    "ALLOW_IP_LIST",
    "ACCOUNT_SETTINGS",
]

_ENFORCEMENTS = [
    ("MASTER_PASSWORD_MINIMUM_LENGTH", 10, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_SPECIAL", 11, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_UPPER", 12, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_LOWER", 13, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_MINIMUM_DIGITS", 14, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_RESTRICT_DAYS_BEFORE_REUSE", 16, "LONG", "LOGIN_SETTINGS"),
    ("REQUIRE_TWO_FACTOR", 20, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("MASTER_PASSWORD_MAXIMUM_DAYS_BEFORE_CHANGE", 22, "LONG", "LOGIN_SETTINGS"),
    ("MASTER_PASSWORD_EXPIRED_AS_OF", 23, "LONG", "LOGIN_SETTINGS"),
    ("MINIMUM_PBKDF2_ITERATIONS", 55, "LONG", "ACCOUNT_SETTINGS"),
    ("MAX_SESSION_LOGIN_TIME", 24, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_PERSISTENT_LOGIN", 25, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("STAY_LOGGED_IN_DEFAULT", 26, "BOOLEAN", "ACCOUNT_ENFORCEMENTS"),
    ("RESTRICT_SHARING_ALL", 30, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_ENTERPRISE", 31, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_EXPORT", 32, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_IMPORT", 111, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_FILE_UPLOAD", 33, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("REQUIRE_ACCOUNT_SHARE", 34, "ACCOUNT_SHARE", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_INCOMING_ALL", 36, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_INCOMING_ENTERPRISE", 37, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTIRCT_SHARING_RECORD_AND_FOLDER", 120, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_SHARING_RECORD_WITH_ATTACHMENTS", 121, "BOOLEAN", "SHARING_AND_UPLOADING"),
    ("RESTRICT_IP_ADDRESSES", 40, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("REQUIRE_DEVICE_APPROVAL", 41, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("REQUIRE_ACCOUNT_RECOVERY_APPROVAL", 42, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_VAULT_IP_ADDRESSES", 43, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("TIP_ZONE_RESTRICT_ALLOWED_IP_RANGES", 44, "IP_WHITELIST", "ALLOW_IP_LIST"),
    ("AUTOMATIC_BACKUP_EVERY_X_DAYS", 45, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_OFFLINE_ACCESS", 46, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("SEND_INVITE_AT_REGISTRATION", 47, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_EMAIL_CHANGE", 48, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_IOS_FINGERPRINT", 49, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_MAC_FINGERPRINT", 50, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_ANDROID_FINGERPRINT", 51, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_WINDOWS_FINGERPRINT", 83, "BOOLEAN", "LOGIN_SETTINGS"),
    ("LOGOUT_TIMER_WEB", 52, "LONG", "ACCOUNT_SETTINGS"),
    ("LOGOUT_TIMER_MOBILE", 53, "LONG", "ACCOUNT_SETTINGS"),
    ("LOGOUT_TIMER_DESKTOP", 54, "LONG", "ACCOUNT_SETTINGS"),
    ("RESTRICT_WEB_VAULT_ACCESS", 60, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_EXTENSIONS_ACCESS", 61, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_ACCESS", 62, " BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_ACCESS", 63, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_IOS_ACCESS", 64, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_ANDROID_ACCESS", 65, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_MOBILE_WINDOWS_PHONE_ACCESS", 66, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_WIN_ACCESS", 67, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_DESKTOP_MAC_ACCESS", 68, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_CHAT_DESKTOP_ACCESS", 84, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_CHAT_MOBILE_ACCESS", 85, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_COMMANDER_ACCESS", 88, "BOOLEAN", "PLATFORM_RESTRICTION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_TEXT", 70, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_GOOGLE", 71, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_DNA", 72, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_DUO", 73, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_RSA", 74, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_WEB", 80, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_MOBILE", 81, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_DURATION_DESKTOP", 82, "TWO_FACTOR_DURATION", "TWO_FACTOR_AUTHENTICATION"),
    ("RESTRICT_TWO_FACTOR_CHANNEL_SECURITY_KEYS", 86, "BOOLEAN", "TWO_FACTOR_AUTHENTICATION"),
    ("TWO_FACTOR_BY_IP", 87, "JSONARRAY"),
    ("RESTRICT_DOMAIN_ACCESS", 90, "STRING", "KEEPER_FILL"),
    ("RESTRICT_DOMAIN_CREATE", 91, "STRING", "KEEPER_FILL"),
    ("RESTRICT_HOVER_LOCKS", 92, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_LOGIN", 93, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_FILL", 94, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_AUTO_SUBMIT", 95, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_SAVE", 96, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_CHANGE", 97, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_AUTO_FILL", 98, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_CREATE_FOLDER", 100, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_CREATE_IDENTITY_PAYMENT_RECORDS", 102, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_CUSTOM_FIELDS", 103, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_NOTES", 104, "BOOLEAN", "VAULT_FEATURES"),
    ("MASK_PASSWORDS_WHILE_EDITING", 105, "BOOLEAN", "VAULT_FEATURES"),
    ("GENERATED_PASSWORD_COMPLEXITY", 106, "STRING", "VAULT_FEATURES"),
    ("GENERATED_SECURITY_QUESTION_COMPLEXITY", 109, "STRING", "VAULT_FEATURES"),
    ("DAYS_BEFORE_DELETED_RECORDS_CLEARED_PERM", 107, "LONG", "VAULT_FEATURES"),
    ("DAYS_BEFORE_DELETED_RECORDS_AUTO_CLEARED", 108, "LONG", "VAULT_FEATURES"),
    ("ALLOW_ALTERNATE_PASSWORDS", 110, "BOOLEAN", "LOGIN_SETTINGS"),
    ("RESTRICT_LINK_SHARING", 122, "BOOLEAN", "SHARING_ENFORCEMENTS"),
    ("DISABLE_SETUP_TOUR", 140, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_PERSONAL_LICENSE", 141, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("DISABLE_ONBOARDING", 142, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("DISALLOW_V2_CLIENTS", 143, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("RESTRICT_IP_AUTOAPPROVAL", 144, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("SEND_BREACH_WATCH_EVENTS", 200, "BOOLEAN", "VAULT_FEATURES"),
    ("RESTRICT_BREACH_WATCH", 201, "BOOLEAN", "VAULT_FEATURES"),
    ("RESEND_ENTERPRISE_INVITE_IN_X_DAYS", 202, "LONG", "ACCOUNT_SETTINGS"),
    ("MASTER_PASSWORD_REENTRY", 203, "JSON"),
    ("RESTRICT_ACCOUNT_RECOVERY", 204, "BOOLEAN", "ACCOUNT_SETTINGS"),
    ("KEEPER_FILL_HOVER_LOCKS", 205, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_AUTO_FILL", 206, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_AUTO_SUBMIT", 207, "TERNARY_DEN", "KEEPER_FILL"),
    ("KEEPER_FILL_MATCH_ON_SUBDOMAIN", 208, "TERNARY_EDN", "KEEPER_FILL"),
    ("RESTRICT_PROMPT_TO_DISABLE", 209, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_HTTP_FILL_WARNING", 210, "BOOLEAN", "KEEPER_FILL"),
    ("RESTRICT_RECORD_TYPES", 211, "RECORD_TYPES", "RECORD_TYPES"),
    ("ALLOW_SECRETS_MANAGER", 212, "BOOLEAN", "VAULT_FEATURES"),
    ("REQUIRE_SELF_DESTRUCT", 213, "BOOLEAN", "ACCOUNT_ENFORCEMENTS"),
    ("KEEPER_FILL_AUTO_SUGGEST", 214, "TERNARY_DEN", "KEEPER_FILL"),
]

def enforcement_list():  # type: () -> List[Tuple[str, str, str]]
    groups = {x[1]: x[0] for x in enumerate(_ENFORCEMENT_GROUPS)}
    enforcements = [(x[3], x[0], x[1], x[2]) for x in _ENFORCEMENTS if len(x) >= 4 and x[3]]
    enforcements.sort(key=lambda x: (groups[x[0]] if x[0] in groups else 100) * 1000 + x[2])
    return [(x[0].title().replace('_', ' '), x[1].lower(), x[3].lower()) for x in enforcements]

ENFORCEMENTS = {}
for e in _ENFORCEMENTS:
    ENFORCEMENTS[e[0].lower()] = e[2].lower()

# OS dependent constants
if sys.platform.startswith('win'):
    OS_WHICH_CMD = 'where'
else:
    OS_WHICH_CMD = 'which'

KEEPER_PUBLIC_HOSTS = {
    'US': 'keepersecurity.com',
    'EU': 'keepersecurity.eu',
    'AU': 'keepersecurity.com.au',
    'CA': 'keepersecurity.ca',
    'JP': 'keepersecurity.jp',
    'GOV': 'govcloud.keepersecurity.us'
}

def get_abbrev_by_host(host):
    # Return abbreviation of the Keeper's public host

    if host.startswith('https:'):
        host = urlparse(host).netloc    # https://keepersecurity.com/api/v2/ --> keepersecurity.com

    keys = [k for k, v in KEEPER_PUBLIC_HOSTS.items() if v == host]
    if keys:
        return keys[0]
    return None

# Messages
# Account Transfer
ACCOUNT_TRANSFER_MSG = """
Your Keeper administrator has enabled the ability to transfer your vault records
in accordance with company operating procedures and policies.
Please acknowledge this change in account settings by typing 'Accept'.
If you do not accept this change by {0}, you will be locked out of your account.
"""
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Script to print the drivers and services from the SYSTEM
# Registry file (REGF)
#
# Copyright (c) 2013, Joachim Metz <joachim.metz@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys

import pyregf

REG_NONE = 0
REG_SZ = 1
REG_EXPAND_SZ = 2
REG_BINARY = 3
REG_DWORD = 4
REG_DWORD_LITTLE_ENDIAN = 4
REG_DWORD_BIG_ENDIAN = 5
REG_LINK = 6
REG_MULTI_SZ = 7
REG_RESOURCE_LIST = 8
REG_FULL_RESOURCE_DESCRIPTOR = 9
REG_RESOURCE_REQUIREMENT_LIST = 10
REG_QWORD = 11

def PrintUsage():
	print "Usage: ./services.py registry_file"
	print ""
	print "   	registry_file: The path of the SYSTEM registry file."
	print ""

def PrintServices( control_set_key ):
	services_key = control_set_key.get_sub_key_by_name( "Services" )

	if services_key:
		print "Control set: {0:s}".format( control_set_key.name )
		print "\tNumber of entries\t: {0:d}".format( services_key.number_of_sub_keys )
		print ""

		for service_key in services_key.sub_keys:
			print "{0:s}".format( service_key.name )

			type_value = service_key.get_value_by_name( "Type" )
			object_name_string = "Object name"

			if type_value:
				type_value = type_value.data_as_integer

				if type_value == 0x00000001:
					type_string = "Kernel device driver"

				elif type_value == 0x00000002:
					type_string = "File system driver"

				elif type_value == 0x00000004:
					type_string = "Adapter arguments"

				elif type_value == 0x00000010:
					type_string = "Stand-alone service"
					object_name_string = "Account name"

				elif type_value == 0x00000020:
					type_string = "Shared service"
					object_name_string = "Account name"

				else:
					if type_value == 0x00000110:
						object_name_string = "Account name"

					type_string = "Unknown 0x{0:08x}".format( type_value )

				print "\tType\t\t\t: {0:s}".format( type_string )

			display_name_value = service_key.get_value_by_name( "DisplayName" )

			if display_name_value:
				if ( display_name_value.type == REG_SZ or display_name_value.type == REG_EXPAND_SZ ):
					print "\tDisplay name\t\t: {0:s}".format( display_name_value.data_as_string )

			description_value = service_key.get_value_by_name( "Description" )

			if description_value:
				print "\tDescription\t\t: {0:s}".format( description_value.data_as_string )

			image_path_value = service_key.get_value_by_name( "ImagePath" )

			if image_path_value:
				print "\tExecutable\t\t: {0:s}".format( image_path_value.data_as_string )

			object_name_value = service_key.get_value_by_name( "ObjectName" )

			if object_name_value:
				print "\t{0:s}\t\t: {1:s}".format( object_name_string, object_name_value.data_as_string )

			start_value = service_key.get_value_by_name( "Start" )

			if start_value:
				start_value = start_value.data_as_integer

				if start_value == 0x00000000:
					start_string = "Boot"

				elif start_value == 0x00000001:
					start_string = "System"

				elif start_value == 0x00000002:
					start_string = "Automatic"

				elif start_value == 0x00000003:
					start_string = "On demand"

				elif start_value == 0x00000004:
					start_string = "Disabled"

				else:
					start_string = "Unknown 0x{0:08x}".format( start_value )

				print "\tStart\t\t\t: {0:s}".format( start_string )

			print ""

def Main( argc, argv ):
	if argc != 2:
		PrintUsage()
		return( 1 )

	regf_file = pyregf.file()
	regf_file.open( argv[1] )

	root_key = regf_file.get_root_key()

	if root_key:
		for control_set_key in root_key.sub_keys:
			if control_set_key.name.startswith( "ControlSet" ):
				PrintServices( control_set_key )
	else:
		print "No root key found."

	regf_file.close()

	return( 0 )

if __name__ == '__main__':
	sys.exit( Main( len( sys.argv ), sys.argv ) )

<commit_msg>Added dfvfs support to services.py<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Script to print the drivers and services from the SYSTEM
# Registry file (REGF)
#
# Copyright (c) 2013, Joachim Metz <joachim.metz@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import logging
import os
import stat
import sys

import dfvfs
import pyregf

from dfvfs.analyzer import analyzer
from dfvfs.lib import definitions
from dfvfs.path import factory as path_spec_factory
from dfvfs.helpers import windows_path_resolver
from dfvfs.resolver import resolver
from dfvfs.vfs import os_file_system
from dfvfs.volume import tsk_volume_system

if dfvfs.__version__ < '20140127':
  raise ImportWarning('shellfolder.py requires dfvfs 20140127 or later.')

if pyregf.get_version() < '20130716':
  raise ImportWarning('shellfolder.py requires pyregf 20130716 or later.')

class CollectorError(Exception):
  """Class that defines collector errors."""

class WindowsVolumeCollector(object):
  """Class that defines a Windows volume collector."""

  _WINDOWS_DIRECTORIES = frozenset([
      u'C:\\Windows',
      u'C:\\WINNT',
      u'C:\\WTSRV',
      u'C:\\WINNT35',
  ])

  def __init__(self):
    """Initializes the Windows volume collector object."""
    super(WindowsVolumeCollector, self).__init__()
    self._file_system = None
    self._path_resolver = None

  def GetWindowsVolumePathSpec(self, source_path):
    """Determines the file system path specification of the Windows volume.

    Args:
      source_path: the source path.

    Returns:
      True if successful or False otherwise.

    Raises:
      CollectorError: if the source path does not exists, or if the source path
                      is not a file or directory, or if the format of or within
                      the source file is not supported.
    """
    if not os.path.exists(source_path):
      raise CollectorError(u'No such source: {0:s}.'.format(source_path))

    stat_info = os.stat(source_path)

    if (not stat.S_ISDIR(stat_info.st_mode) and
        not stat.S_ISREG(stat_info.st_mode)):
      raise CollectorError(
          u'Unsupported source: {0:s} not a file or directory.'.format(
              source_path))

    path_spec = path_spec_factory.Factory.NewPathSpec(
        definitions.TYPE_INDICATOR_OS, location=source_path)

    if stat.S_ISREG(stat_info.st_mode):
      type_indicators = analyzer.Analyzer.GetStorageMediaImageTypeIndicators(
          path_spec)

      if len(type_indicators) > 1:
        raise CollectorError((
            u'Unsupported source: {0:s} found more than one storage media '
            u'image types.').format(source_path))

      if len(type_indicators) == 1:
        path_spec = path_spec_factory.Factory.NewPathSpec(
            type_indicators[0], parent=path_spec)

      type_indicators = analyzer.Analyzer.GetVolumeSystemTypeIndicators(
          path_spec)

      if len(type_indicators) > 1:
        raise CollectorError((
            u'Unsupported source: {0:s} found more than one volume system '
            u'types.').format(source_path))

      if len(type_indicators) == 1:
        if type_indicators[0] == definitions.TYPE_INDICATOR_TSK_PARTITION:
          vs_path_spec = path_spec_factory.Factory.NewPathSpec(
              type_indicators[0], location='/', parent=path_spec)

          volume_system = tsk_volume_system.TSKVolumeSystem()
          volume_system.Open(vs_path_spec)

          result = False

          for volume in volume_system.volumes:
            if not hasattr(volume, 'identifier'):
              continue

            volume_location = u'/{0:s}'.format(volume.identifier)
            volume_path_spec = path_spec_factory.Factory.NewPathSpec(
                type_indicators[0], location=volume_location, parent=path_spec)

            fs_path_spec = path_spec_factory.Factory.NewPathSpec(
                definitions.TYPE_INDICATOR_TSK, location=u'/',
                parent=volume_path_spec)
            file_system = resolver.Resolver.OpenFileSystem(fs_path_spec)

            if file_system is None:
              continue

            path_resolver = windows_path_resolver.WindowsPathResolver(
                file_system, volume_path_spec)

            for windows_path in self._WINDOWS_DIRECTORIES:
              windows_path_spec = path_resolver.ResolvePath(windows_path)

              result = windows_path_spec is not None

              if result:
                path_spec = volume_path_spec
                break

            if result:
              break

          if not result:
            return False

        elif type_indicators[0] != definitions.TYPE_INDICATOR_VSHADOW:
          raise CollectorError((
              u'Unsupported source: {0:s} found unsupported volume system '
              u'type: {1:s}.').format(source_path, type_indicators[0]))

      type_indicators = analyzer.Analyzer.GetFileSystemTypeIndicators(
          path_spec)

      if len(type_indicators) == 0:
        return False

      if len(type_indicators) > 1:
        raise CollectorError((
            u'Unsupported source: {0:s} found more than one file system '
            u'types.').format(source_path))

      if type_indicators[0] != definitions.TYPE_INDICATOR_TSK:
        raise CollectorError((
            u'Unsupported source: {0:s} found unsupported file system '
            u'type: {1:s}.').format(source_path, type_indicators[0]))

      fs_path_spec = path_spec_factory.Factory.NewPathSpec(
          definitions.TYPE_INDICATOR_TSK, location=u'/',
          parent=path_spec)
      self._file_system = resolver.Resolver.OpenFileSystem(fs_path_spec)

    elif stat.S_ISDIR(stat_info.st_mode):
      self._file_system = os_file_system.OSFileSystem()

    self._path_resolver = windows_path_resolver.WindowsPathResolver(
        self._file_system, path_spec)

    return True

  def OpenFile(self, windows_path):
    """Opens the file specificed by the Windows path.

    Args:
      windows_path: the Windows path to the file.

    Returns:
      The file-like object (instance of dfvfs.FileIO) or None if
      the file does not exist.
    """
    path_spec = self._path_resolver.ResolvePath(windows_path)
    if path_spec is None:
      return None

    return resolver.Resolver.OpenFileObject(path_spec)

class WindowsServiceCollector(WindowsVolumeCollector):
  """Class that defines a Windows service collector."""

  _REGISTRY_FILENAME_SYSTEM = u'C:\\Windows\\System32\\config\\SYSTEM'

  def __init__(self):
    """Initializes the Windows service collector object."""
    super(WindowsServiceCollector, self).__init__()
    self.found_services_key = False

  def _OpenRegistryFile(self, windows_path):
    """Opens the registry file specificed by the Windows path.

    Args:
      windows_path: the Windows path to the Registry file.

    Returns:
      The Registry file (instance of RegistryFile) or None.
    """
    file_object = self.OpenFile(windows_path)
    if file_object is None:
      return None

    registry_file = RegistryFile()
    registry_file.Open(file_object)
    return registry_file

  def _CollectWindowsServicesFromKey(self, output_writer, services_key):
    """Collects the Windows services from a services Registry key.

    Args:
      output_writer: the output writer object.
      services_key: the services Registry key (instance of pyregf.key).
    """
    print u'\tNumber of entries\t: {0:d}'.format(
        services_key.number_of_sub_keys)
    print u''

    for service_key in services_key.sub_keys:
      print u'{0:s}'.format(service_key.name)

      type_value = service_key.get_value_by_name('Type')
      object_name_string = 'Object name'

      if type_value:
        type_value = type_value.data_as_integer

        if type_value == 0x00000001:
          type_string = 'Kernel device driver'

        elif type_value == 0x00000002:
          type_string = 'File system driver'

        elif type_value == 0x00000004:
          type_string = 'Adapter arguments'

        elif type_value == 0x00000010:
          type_string = 'Stand-alone service'
          object_name_string = 'Account name'

        elif type_value == 0x00000020:
          type_string = 'Shared service'
          object_name_string = 'Account name'

        else:
          if type_value == 0x00000110:
            object_name_string = 'Account name'

          type_string = 'Unknown 0x{0:08x}'.format(type_value)

        print u'\tType\t\t\t: {0:s}'.format(type_string)

      display_name_value = service_key.get_value_by_name('DisplayName')

      if display_name_value:
        if display_name_value.type in [
            RegistryFile.REG_SZ, RegistryFile.REG_EXPAND_SZ]:
          print u'\tDisplay name\t\t: {0:s}'.format(
              display_name_value.data_as_string)

      description_value = service_key.get_value_by_name('Description')

      if description_value:
        print u'\tDescription\t\t: {0:s}'.format(
            description_value.data_as_string)

      image_path_value = service_key.get_value_by_name('ImagePath')

      if image_path_value:
        print u'\tExecutable\t\t: {0:s}'.format(image_path_value.data_as_string)

      object_name_value = service_key.get_value_by_name('ObjectName')

      if object_name_value:
        print u'\t{0:s}\t\t: {1:s}'.format(
            object_name_string, object_name_value.data_as_string)

      start_value = service_key.get_value_by_name('Start')

      if start_value:
        start_value = start_value.data_as_integer

        if start_value == 0x00000000:
          start_string = 'Boot'

        elif start_value == 0x00000001:
          start_string = 'System'

        elif start_value == 0x00000002:
          start_string = 'Automatic'

        elif start_value == 0x00000003:
          start_string = 'On demand'

        elif start_value == 0x00000004:
          start_string = 'Disabled'

        else:
          start_string = 'Unknown 0x{0:08x}'.format(start_value)

        print u'\tStart\t\t\t: {0:s}'.format(start_string)

      print u''

  def CollectWindowsServices(self, output_writer):
    """Collects the Windows services from the SYSTEM Registry file.

    Args:
      output_writer: the output writer object.
    """
    registry_file = self._OpenRegistryFile(
        self._REGISTRY_FILENAME_SYSTEM)

    root_key = registry_file.GetRootKey()

    if root_key:
      for control_set_key in root_key.sub_keys:
        if control_set_key.name.startswith('ControlSet'):
          services_key = control_set_key.get_sub_key_by_name('Services')
          if services_key:
            self.found_services_key = True

            print u'Control set: {0:s}'.format(control_set_key.name)
            self._CollectWindowsServicesFromKey(output_writer, services_key)

    registry_file.Close()

class RegistryFile(object):
  """Class that defines a Windows Registry file."""

  _CLASS_IDENTIFIERS_KEY_PATH = 'Classes\\CLSID'

  REG_NONE = 0
  REG_SZ = 1
  REG_EXPAND_SZ = 2
  REG_BINARY = 3
  REG_DWORD = 4
  REG_DWORD_LITTLE_ENDIAN = 4
  REG_DWORD_BIG_ENDIAN = 5
  REG_LINK = 6
  REG_MULTI_SZ = 7
  REG_RESOURCE_LIST = 8
  REG_FULL_RESOURCE_DESCRIPTOR = 9
  REG_RESOURCE_REQUIREMENT_LIST = 10
  REG_QWORD = 11

  def __init__(self, ascii_codepage='cp1252'):
    """Initializes the Windows Registry file.

    Args:
      ascii_codepage: optional ASCII string codepage. The default is cp1252
                      (or windows-1252).
    """
    super(RegistryFile, self).__init__()
    self._file_object = None
    self._regf_file = pyregf.file()
    self._regf_file.set_ascii_codepage(ascii_codepage)

  def Open(self, file_object):
    """Opens the Windows Registry file using a file-like object.

    Args:
      file_object: the file-like object.

    Returns:
      A boolean containing True if successful or False if not.
    """
    self._file_object = file_object
    self._regf_file.open_file_object(self._file_object)
    return True

  def Close(self):
    """Closes the Windows Registry file."""
    self._regf_file.close()
    self._file_object.close()
    self._file_object = None

  def GetRootKey(self):
    """Retrieves the root keys.

    Yields:
      A Registry key (instance of pyregf.key).
    """
    return self._regf_file.get_root_key()

class StdoutWriter(object):
  """Class that defines a stdout output writer."""

  def Open(self):
    """Opens the output writer object.

    Returns:
      A boolean containing True if successful or False if not.
    """
    return True

  def Close(self):
    """Closes the output writer object."""
    pass

def Main():
  """The main program function.

  Returns:
    A boolean containing True if successful or False if not.
  """
  args_parser = argparse.ArgumentParser(description=(
      'Extract the services information from a SYSTEM '
      ' Registry File (REGF).'))

  args_parser.add_argument(
      'source', nargs='?', action='store', metavar='/mnt/c/',
      default=None, help=('path of the volume containing C:\\Windows or the '
                          'filename of a storage media image containing the '
                          'C:\\Windows directory.'))

  options = args_parser.parse_args()

  if not options.source:
    print u'Source value is missing.'
    print u''
    args_parser.print_help()
    print u''
    return False

  logging.basicConfig(
      level=logging.INFO, format=u'[%(levelname)s] %(message)s')

  output_writer = StdoutWriter()

  if not output_writer.Open():
    print u'Unable to open output writer.'
    print u''
    return False

  collector = WindowsServiceCollector()

  if not collector.GetWindowsVolumePathSpec(options.source):
    print (
        u'Unable to retrieve the volume with the Windows directory from: '
        u'{0:s}.').format(options.source)
    print ''
    return False

  collector.CollectWindowsServices(output_writer)
  output_writer.Close()

  if not collector.found_services_key:
    print u'No services key found.'

  return True

if __name__ == '__main__':
  if not Main():
    sys.exit(1)
  else:
    sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys
import cv2

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
    import keras_retinanet.bin  # noqa: F401
    __package__ = "keras_retinanet.bin"

# Change these to absolute imports if you copy this script outside the keras_retinanet package.
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..preprocessing.kitti import KittiGenerator
from ..preprocessing.open_images import OpenImagesGenerator
from ..utils.keras_version import check_keras_version
from ..utils.transform import random_transform_generator
from ..utils.visualization import draw_annotations, draw_boxes
from ..utils.anchors import anchors_for_shape, compute_gt_annotations
from ..utils.config import read_config_file, parse_anchor_parameters

def create_generator(args):
    """ Create the data generators.

    Args:
        args: parseargs arguments object.
    """
    # create random transform generator for augmenting training data
    transform_generator = random_transform_generator(
        min_rotation=-0.1,
        max_rotation=0.1,
        min_translation=(-0.1, -0.1),
        max_translation=(0.1, 0.1),
        min_shear=-0.1,
        max_shear=0.1,
        min_scaling=(0.9, 0.9),
        max_scaling=(1.1, 1.1),
        flip_x_chance=0.5,
        flip_y_chance=0.5,
    )

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        generator = CocoGenerator(
            args.coco_path,
            args.coco_set,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'pascal':
        generator = PascalVocGenerator(
            args.pascal_path,
            args.pascal_set,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'csv':
        generator = CSVGenerator(
            args.annotations,
            args.classes,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'oid':
        generator = OpenImagesGenerator(
            args.main_dir,
            subset=args.subset,
            version=args.version,
            labels_filter=args.labels_filter,
            parent_label=args.parent_label,
            annotation_cache_dir=args.annotation_cache_dir,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'kitti':
        generator = KittiGenerator(
            args.kitti_path,
            subset=args.subset,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return generator

def parse_args(args):
    """ Parse the arguments.
    """
    parser     = argparse.ArgumentParser(description='Debug script for a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path',  help='Path to dataset directory (ie. /tmp/COCO).')
    coco_parser.add_argument('--coco-set', help='Name of the set to show (defaults to val2017).', default='val2017')

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')
    pascal_parser.add_argument('--pascal-set',  help='Name of the set to show (defaults to test).', default='test')

    kitti_parser = subparsers.add_parser('kitti')
    kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')
    kitti_parser.add_argument('subset', help='Argument for loading a subset from train/val.')

    def csv_list(string):
        return string.split(',')

    oid_parser = subparsers.add_parser('oid')
    oid_parser.add_argument('main_dir', help='Path to dataset directory.')
    oid_parser.add_argument('subset', help='Argument for loading a subset from train/validation/test.')
    oid_parser.add_argument('--version',  help='The current dataset version is v4.', default='v4')
    oid_parser.add_argument('--labels-filter',  help='A list of labels to filter.', type=csv_list, default=None)
    oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')
    oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')
    csv_parser.add_argument('classes',     help='Path to a CSV file containing class label mapping.')

    parser.add_argument('-l', '--loop', help='Loop forever, even if the dataset is exhausted.', action='store_true')
    parser.add_argument('--no-resize', help='Disable image resizing.', dest='resize', action='store_false')
    parser.add_argument('--anchors', help='Show positive anchors on the image.', action='store_true')
    parser.add_argument('--annotations', help='Show annotations on the image. Green annotations have anchors, red annotations don\'t and therefore don\'t contribute to training.', action='store_true')
    parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')
    parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)
    parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)
    parser.add_argument('--config', help='Path to a configuration parameters .ini file.')

    return parser.parse_args(args)

def run(generator, args, anchor_params):
    """ Main loop.

    Args
        generator: The generator to debug.
        args: parseargs args object.
    """
    # display images, one at a time
    for i in range(generator.size()):
        # load the data
        image       = generator.load_image(i)
        annotations = generator.load_annotations(i)
        if len(annotations['labels']) > 0 :
            # apply random transformations
            if args.random_transform:
                image, annotations = generator.random_transform_group_entry(image, annotations)

            # resize the image and annotations
            if args.resize:
                image, image_scale = generator.resize_image(image)
                annotations['bboxes'] *= image_scale

            anchors = anchors_for_shape(image.shape, anchor_params=anchor_params)
            positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])

            # draw anchors on the image
            if args.anchors:
                draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)

            # draw annotations on the image
            if args.annotations:
                # draw annotations in red
                draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)

                # draw regressed anchors in green to override most red annotations
                # result is that annotations without anchors are red, with anchors are green
                draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))

        key = cv2.waitKey(1)
        while True:
            key = cv2.waitKey(1)
            cv2.imshow('Image', image)
            # press n for next image
            if key == ord('n'):
                i += 1
                break
            # press b for previous image
            if key == ord('b'):
                i -= 1
                break
            # press q to quit
            if key == ord('q'):
                return False
    return True

def main(args=None):
    # parse arguments
    if args is None:
        args = sys.argv[1:]
    args = parse_args(args)

    # make sure keras is the minimum required version
    check_keras_version()

    # create the generator
    generator = create_generator(args)

    # sorting list of images for cycling through dataset
    generator.image_names = sorted(generator.image_names)

    # optionally load config parameters
    if args.config:
        args.config = read_config_file(args.config)

    # optionally load anchor parameters
    anchor_params = None
    if args.config and 'anchor_parameters' in args.config:
        anchor_params = parse_anchor_parameters(args.config)

    # create the display window
    cv2.namedWindow('Image', cv2.WINDOW_NORMAL)

    if args.loop:
        while run(generator, args, anchor_params=anchor_params):
            pass
    else:
        run(generator, args, anchor_params=anchor_params)

if __name__ == '__main__':
    main()
<commit_msg>Change debug.py keybindings to arrow keys<commit_after>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys
import cv2

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
    import keras_retinanet.bin  # noqa: F401
    __package__ = "keras_retinanet.bin"

# Change these to absolute imports if you copy this script outside the keras_retinanet package.
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..preprocessing.kitti import KittiGenerator
from ..preprocessing.open_images import OpenImagesGenerator
from ..utils.keras_version import check_keras_version
from ..utils.transform import random_transform_generator
from ..utils.visualization import draw_annotations, draw_boxes
from ..utils.anchors import anchors_for_shape, compute_gt_annotations
from ..utils.config import read_config_file, parse_anchor_parameters

def create_generator(args):
    """ Create the data generators.

    Args:
        args: parseargs arguments object.
    """
    # create random transform generator for augmenting training data
    transform_generator = random_transform_generator(
        min_rotation=-0.1,
        max_rotation=0.1,
        min_translation=(-0.1, -0.1),
        max_translation=(0.1, 0.1),
        min_shear=-0.1,
        max_shear=0.1,
        min_scaling=(0.9, 0.9),
        max_scaling=(1.1, 1.1),
        flip_x_chance=0.5,
        flip_y_chance=0.5,
    )

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        generator = CocoGenerator(
            args.coco_path,
            args.coco_set,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'pascal':
        generator = PascalVocGenerator(
            args.pascal_path,
            args.pascal_set,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'csv':
        generator = CSVGenerator(
            args.annotations,
            args.classes,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'oid':
        generator = OpenImagesGenerator(
            args.main_dir,
            subset=args.subset,
            version=args.version,
            labels_filter=args.labels_filter,
            parent_label=args.parent_label,
            annotation_cache_dir=args.annotation_cache_dir,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'kitti':
        generator = KittiGenerator(
            args.kitti_path,
            subset=args.subset,
            transform_generator=transform_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return generator

def parse_args(args):
    """ Parse the arguments.
    """
    parser     = argparse.ArgumentParser(description='Debug script for a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path',  help='Path to dataset directory (ie. /tmp/COCO).')
    coco_parser.add_argument('--coco-set', help='Name of the set to show (defaults to val2017).', default='val2017')

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')
    pascal_parser.add_argument('--pascal-set',  help='Name of the set to show (defaults to test).', default='test')

    kitti_parser = subparsers.add_parser('kitti')
    kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')
    kitti_parser.add_argument('subset', help='Argument for loading a subset from train/val.')

    def csv_list(string):
        return string.split(',')

    oid_parser = subparsers.add_parser('oid')
    oid_parser.add_argument('main_dir', help='Path to dataset directory.')
    oid_parser.add_argument('subset', help='Argument for loading a subset from train/validation/test.')
    oid_parser.add_argument('--version',  help='The current dataset version is v4.', default='v4')
    oid_parser.add_argument('--labels-filter',  help='A list of labels to filter.', type=csv_list, default=None)
    oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')
    oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')
    csv_parser.add_argument('classes',     help='Path to a CSV file containing class label mapping.')

    parser.add_argument('-l', '--loop', help='Loop forever, even if the dataset is exhausted.', action='store_true')
    parser.add_argument('--no-resize', help='Disable image resizing.', dest='resize', action='store_false')
    parser.add_argument('--anchors', help='Show positive anchors on the image.', action='store_true')
    parser.add_argument('--annotations', help='Show annotations on the image. Green annotations have anchors, red annotations don\'t and therefore don\'t contribute to training.', action='store_true')
    parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')
    parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)
    parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)
    parser.add_argument('--config', help='Path to a configuration parameters .ini file.')

    return parser.parse_args(args)

def run(generator, args, anchor_params):
    """ Main loop.

    Args
        generator: The generator to debug.
        args: parseargs args object.
    """
    # display images, one at a time
    for i in range(generator.size()):
        # load the data
        image       = generator.load_image(i)
        annotations = generator.load_annotations(i)
        if len(annotations['labels']) > 0 :
            # apply random transformations
            if args.random_transform:
                image, annotations = generator.random_transform_group_entry(image, annotations)

            # resize the image and annotations
            if args.resize:
                image, image_scale = generator.resize_image(image)
                annotations['bboxes'] *= image_scale

            anchors = anchors_for_shape(image.shape, anchor_params=anchor_params)
            positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])

            # draw anchors on the image
            if args.anchors:
                draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)

            # draw annotations on the image
            if args.annotations:
                # draw annotations in red
                draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)

                # draw regressed anchors in green to override most red annotations
                # result is that annotations without anchors are red, with anchors are green
                draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))

        key = cv2.waitKey(1)
        cv2.imshow('Image', image)
        # note that the right and left keybindings are probably different for windows
        # press right for next image
        if key == 83:
            i += 1
        # press left for previous image
        if key == 81:
            i -= 1
        # press q to quit
        if key == ord('q'):
            return False
    return True

def main(args=None):
    # parse arguments
    if args is None:
        args = sys.argv[1:]
    args = parse_args(args)

    # make sure keras is the minimum required version
    check_keras_version()

    # create the generator
    generator = create_generator(args)

    # optionally load config parameters
    if args.config:
        args.config = read_config_file(args.config)

    # optionally load anchor parameters
    anchor_params = None
    if args.config and 'anchor_parameters' in args.config:
        anchor_params = parse_anchor_parameters(args.config)

    # create the display window
    cv2.namedWindow('Image', cv2.WINDOW_NORMAL)

    if args.loop:
        while run(generator, args, anchor_params=anchor_params):
            pass
    else:
        run(generator, args, anchor_params=anchor_params)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys
import cv2

# Set keycodes for changing images
# 81, 83 are left and right arrows on linux in Ascii code (probably not needed)
# 65361, 65363 are left and right arrows in linux
# 2424832, 2555904 are left and right arrows on Windows
# 110, 109 are 'n' and 'm' on mac, windows, linux
# (unfortunately arrow keys not picked up on mac)
leftkeys = (81, 110, 65361, 2424832)
rightkeys = (83, 109, 65363, 2555904)

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
    import keras_retinanet.bin  # noqa: F401
    __package__ = "keras_retinanet.bin"

# Change these to absolute imports if you copy this script outside the keras_retinanet package.
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..preprocessing.kitti import KittiGenerator
from ..preprocessing.open_images import OpenImagesGenerator
from ..utils.keras_version import check_keras_version
from ..utils.transform import random_transform_generator
from ..utils.visualization import draw_annotations, draw_boxes, draw_caption
from ..utils.anchors import anchors_for_shape, compute_gt_annotations
from ..utils.config import read_config_file, parse_anchor_parameters
from ..utils.image import random_visual_effect_generator

def create_generator(args):
    """ Create the data generators.

    Args:
        args: parseargs arguments object.
    """
    # create random transform generator for augmenting training data
    transform_generator = random_transform_generator(
        min_rotation=-0.1,
        max_rotation=0.1,
        min_translation=(-0.1, -0.1),
        max_translation=(0.1, 0.1),
        min_shear=-0.1,
        max_shear=0.1,
        min_scaling=(0.9, 0.9),
        max_scaling=(1.1, 1.1),
        flip_x_chance=0.5,
        flip_y_chance=0.5,
    )

    visual_effect_generator = random_visual_effect_generator(
        contrast_range=(0.9, 1.1),
        brightness_range=(-.1, .1),
        hue_range=(-0.05, 0.05),
        saturation_range=(0.95, 1.05)
    )

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        generator = CocoGenerator(
            args.coco_path,
            args.coco_set,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'pascal':
        generator = PascalVocGenerator(
            args.pascal_path,
            args.pascal_set,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'csv':
        generator = CSVGenerator(
            args.annotations,
            args.classes,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'oid':
        generator = OpenImagesGenerator(
            args.main_dir,
            subset=args.subset,
            version=args.version,
            labels_filter=args.labels_filter,
            parent_label=args.parent_label,
            annotation_cache_dir=args.annotation_cache_dir,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'kitti':
        generator = KittiGenerator(
            args.kitti_path,
            subset=args.subset,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return generator

def parse_args(args):
    """ Parse the arguments.
    """
    parser     = argparse.ArgumentParser(description='Debug script for a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path',  help='Path to dataset directory (ie. /tmp/COCO).')
    coco_parser.add_argument('--coco-set', help='Name of the set to show (defaults to val2017).', default='val2017')

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')
    pascal_parser.add_argument('--pascal-set',  help='Name of the set to show (defaults to test).', default='test')

    kitti_parser = subparsers.add_parser('kitti')
    kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')
    kitti_parser.add_argument('subset', help='Argument for loading a subset from train/val.')

    def csv_list(string):
        return string.split(',')

    oid_parser = subparsers.add_parser('oid')
    oid_parser.add_argument('main_dir', help='Path to dataset directory.')
    oid_parser.add_argument('subset', help='Argument for loading a subset from train/validation/test.')
    oid_parser.add_argument('--version',  help='The current dataset version is v4.', default='v4')
    oid_parser.add_argument('--labels-filter',  help='A list of labels to filter.', type=csv_list, default=None)
    oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')
    oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')
    csv_parser.add_argument('classes',     help='Path to a CSV file containing class label mapping.')

    parser.add_argument('--no-resize', help='Disable image resizing.', dest='resize', action='store_false')
    parser.add_argument('--anchors', help='Show positive anchors on the image.', action='store_true')
    parser.add_argument('--display-name', help='Display image name on the bottom left corner.', action='store_true')
    parser.add_argument('--annotations', help='Show annotations on the image. Green annotations have anchors, red annotations don\'t and therefore don\'t contribute to training.', action='store_true')
    parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')
    parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)
    parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)
    parser.add_argument('--config', help='Path to a configuration parameters .ini file.')
    parser.add_argument('--no-gui', help='Do not open a GUI window. Save images to an output directory instead.', action='store_true')
    parser.add_argument('--output-dir', help='The output directory to save images to if --no-gui is specified.', default='.')
    parser.add_argument('--flatten-output', help='Flatten the folder structure of saved output images into a single folder.', action='store_true')

    return parser.parse_args(args)

def run(generator, args, anchor_params):
    """ Main loop.

    Args
        generator: The generator to debug.
        args: parseargs args object.
    """
    # display images, one at a time
    i = 0
    while True:
        # load the data
        image       = generator.load_image(i)
        annotations = generator.load_annotations(i)
        if len(annotations['labels']) > 0 :
            # apply random transformations
            if args.random_transform:
                image, annotations = generator.random_transform_group_entry(image, annotations)
                image, annotations = generator.random_visual_effect_group_entry(image, annotations)

            # resize the image and annotations
            if args.resize:
                image, image_scale = generator.resize_image(image)
                annotations['bboxes'] *= image_scale

            anchors = anchors_for_shape(image.shape, anchor_params=anchor_params)
            positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])

            # draw anchors on the image
            if args.anchors:
                draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)

            # draw annotations on the image
            if args.annotations:
                # draw annotations in red
                draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)

                # draw regressed anchors in green to override most red annotations
                # result is that annotations without anchors are red, with anchors are green
                draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))

            # display name on the image
            if args.display_name:
                draw_caption(image, [0, image.shape[0]], os.path.basename(generator.image_path(i)))

        # write to file and advance if no-gui selected
        if args.no_gui:
            output_path = make_output_path(args.output_dir, generator.image_path(i), flatten=args.flatten_output)
            os.makedirs(os.path.dirname(output_path), exist_ok=True)
            cv2.imwrite(output_path, image)
            i += 1
            if i == generator.size():  # have written all images
                break
            else:
                continue

        # if we are using the GUI, then show an image
        cv2.imshow('Image', image)
        key = cv2.waitKeyEx()

        # press right for next image and left for previous (linux or windows, doesn't work for macOS)
        # if you run macOS, press "n" or "m" (will also work on linux and windows)

        if key in rightkeys:
            i = (i + 1) % generator.size()
        if key in leftkeys:
            i -= 1
            if i < 0:
                i = generator.size() - 1

        # press q or Esc to quit
        if (key == ord('q')) or (key == 27):
            return False

    return True

def make_output_path(output_dir, image_path, flatten = False):
    """ Compute the output path for a debug image. """

    # If the output hierarchy is flattened to a single folder, throw away all leading folders.
    if flatten:
        path = os.path.basename(image_path)

    # Otherwise, make sure absolute paths are taken relative to the filesystem root.
    else:
        # Make sure to drop drive letters on Windows, otherwise relpath wil fail.
        _, path = os.path.splitdrive(image_path)
        if os.path.isabs(path):
            path = path.relpath(path, '/')

    # In all cases, append "_debug" to the filename, before the extension.
    base, extension = os.path.splitext(path)
    path = base + "_debug" + extension

    # Finally, join the whole thing to the output directory.
    return os.path.join(output_dir, path)

def main(args=None):
    # parse arguments
    if args is None:
        args = sys.argv[1:]
    args = parse_args(args)

    # make sure keras is the minimum required version
    check_keras_version()

    # create the generator
    generator = create_generator(args)

    # optionally load config parameters
    if args.config:
        args.config = read_config_file(args.config)

    # optionally load anchor parameters
    anchor_params = None
    if args.config and 'anchor_parameters' in args.config:
        anchor_params = parse_anchor_parameters(args.config)

    # create the display window if necessary
    if not args.no_gui:
        cv2.namedWindow('Image', cv2.WINDOW_NORMAL)

    run(generator, args, anchor_params=anchor_params)

if __name__ == '__main__':
    main()
<commit_msg>debug.py: Fix unqualified use of os.path module.<commit_after>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys
import cv2

# Set keycodes for changing images
# 81, 83 are left and right arrows on linux in Ascii code (probably not needed)
# 65361, 65363 are left and right arrows in linux
# 2424832, 2555904 are left and right arrows on Windows
# 110, 109 are 'n' and 'm' on mac, windows, linux
# (unfortunately arrow keys not picked up on mac)
leftkeys = (81, 110, 65361, 2424832)
rightkeys = (83, 109, 65363, 2555904)

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
    import keras_retinanet.bin  # noqa: F401
    __package__ = "keras_retinanet.bin"

# Change these to absolute imports if you copy this script outside the keras_retinanet package.
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..preprocessing.kitti import KittiGenerator
from ..preprocessing.open_images import OpenImagesGenerator
from ..utils.keras_version import check_keras_version
from ..utils.transform import random_transform_generator
from ..utils.visualization import draw_annotations, draw_boxes, draw_caption
from ..utils.anchors import anchors_for_shape, compute_gt_annotations
from ..utils.config import read_config_file, parse_anchor_parameters
from ..utils.image import random_visual_effect_generator

def create_generator(args):
    """ Create the data generators.

    Args:
        args: parseargs arguments object.
    """
    # create random transform generator for augmenting training data
    transform_generator = random_transform_generator(
        min_rotation=-0.1,
        max_rotation=0.1,
        min_translation=(-0.1, -0.1),
        max_translation=(0.1, 0.1),
        min_shear=-0.1,
        max_shear=0.1,
        min_scaling=(0.9, 0.9),
        max_scaling=(1.1, 1.1),
        flip_x_chance=0.5,
        flip_y_chance=0.5,
    )

    visual_effect_generator = random_visual_effect_generator(
        contrast_range=(0.9, 1.1),
        brightness_range=(-.1, .1),
        hue_range=(-0.05, 0.05),
        saturation_range=(0.95, 1.05)
    )

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        generator = CocoGenerator(
            args.coco_path,
            args.coco_set,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'pascal':
        generator = PascalVocGenerator(
            args.pascal_path,
            args.pascal_set,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'csv':
        generator = CSVGenerator(
            args.annotations,
            args.classes,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'oid':
        generator = OpenImagesGenerator(
            args.main_dir,
            subset=args.subset,
            version=args.version,
            labels_filter=args.labels_filter,
            parent_label=args.parent_label,
            annotation_cache_dir=args.annotation_cache_dir,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    elif args.dataset_type == 'kitti':
        generator = KittiGenerator(
            args.kitti_path,
            subset=args.subset,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            image_min_side=args.image_min_side,
            image_max_side=args.image_max_side,
            config=args.config
        )
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return generator

def parse_args(args):
    """ Parse the arguments.
    """
    parser     = argparse.ArgumentParser(description='Debug script for a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path',  help='Path to dataset directory (ie. /tmp/COCO).')
    coco_parser.add_argument('--coco-set', help='Name of the set to show (defaults to val2017).', default='val2017')

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')
    pascal_parser.add_argument('--pascal-set',  help='Name of the set to show (defaults to test).', default='test')

    kitti_parser = subparsers.add_parser('kitti')
    kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')
    kitti_parser.add_argument('subset', help='Argument for loading a subset from train/val.')

    def csv_list(string):
        return string.split(',')

    oid_parser = subparsers.add_parser('oid')
    oid_parser.add_argument('main_dir', help='Path to dataset directory.')
    oid_parser.add_argument('subset', help='Argument for loading a subset from train/validation/test.')
    oid_parser.add_argument('--version',  help='The current dataset version is v4.', default='v4')
    oid_parser.add_argument('--labels-filter',  help='A list of labels to filter.', type=csv_list, default=None)
    oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')
    oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')
    csv_parser.add_argument('classes',     help='Path to a CSV file containing class label mapping.')

    parser.add_argument('--no-resize', help='Disable image resizing.', dest='resize', action='store_false')
    parser.add_argument('--anchors', help='Show positive anchors on the image.', action='store_true')
    parser.add_argument('--display-name', help='Display image name on the bottom left corner.', action='store_true')
    parser.add_argument('--annotations', help='Show annotations on the image. Green annotations have anchors, red annotations don\'t and therefore don\'t contribute to training.', action='store_true')
    parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')
    parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)
    parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)
    parser.add_argument('--config', help='Path to a configuration parameters .ini file.')
    parser.add_argument('--no-gui', help='Do not open a GUI window. Save images to an output directory instead.', action='store_true')
    parser.add_argument('--output-dir', help='The output directory to save images to if --no-gui is specified.', default='.')
    parser.add_argument('--flatten-output', help='Flatten the folder structure of saved output images into a single folder.', action='store_true')

    return parser.parse_args(args)

def run(generator, args, anchor_params):
    """ Main loop.

    Args
        generator: The generator to debug.
        args: parseargs args object.
    """
    # display images, one at a time
    i = 0
    while True:
        # load the data
        image       = generator.load_image(i)
        annotations = generator.load_annotations(i)
        if len(annotations['labels']) > 0 :
            # apply random transformations
            if args.random_transform:
                image, annotations = generator.random_transform_group_entry(image, annotations)
                image, annotations = generator.random_visual_effect_group_entry(image, annotations)

            # resize the image and annotations
            if args.resize:
                image, image_scale = generator.resize_image(image)
                annotations['bboxes'] *= image_scale

            anchors = anchors_for_shape(image.shape, anchor_params=anchor_params)
            positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])

            # draw anchors on the image
            if args.anchors:
                draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)

            # draw annotations on the image
            if args.annotations:
                # draw annotations in red
                draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)

                # draw regressed anchors in green to override most red annotations
                # result is that annotations without anchors are red, with anchors are green
                draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))

            # display name on the image
            if args.display_name:
                draw_caption(image, [0, image.shape[0]], os.path.basename(generator.image_path(i)))

        # write to file and advance if no-gui selected
        if args.no_gui:
            output_path = make_output_path(args.output_dir, generator.image_path(i), flatten=args.flatten_output)
            os.makedirs(os.path.dirname(output_path), exist_ok=True)
            cv2.imwrite(output_path, image)
            i += 1
            if i == generator.size():  # have written all images
                break
            else:
                continue

        # if we are using the GUI, then show an image
        cv2.imshow('Image', image)
        key = cv2.waitKeyEx()

        # press right for next image and left for previous (linux or windows, doesn't work for macOS)
        # if you run macOS, press "n" or "m" (will also work on linux and windows)

        if key in rightkeys:
            i = (i + 1) % generator.size()
        if key in leftkeys:
            i -= 1
            if i < 0:
                i = generator.size() - 1

        # press q or Esc to quit
        if (key == ord('q')) or (key == 27):
            return False

    return True

def make_output_path(output_dir, image_path, flatten = False):
    """ Compute the output path for a debug image. """

    # If the output hierarchy is flattened to a single folder, throw away all leading folders.
    if flatten:
        path = os.path.basename(image_path)

    # Otherwise, make sure absolute paths are taken relative to the filesystem root.
    else:
        # Make sure to drop drive letters on Windows, otherwise relpath wil fail.
        _, path = os.path.splitdrive(image_path)
        if os.path.isabs(path):
            path = os.path.relpath(path, '/')

    # In all cases, append "_debug" to the filename, before the extension.
    base, extension = os.path.splitext(path)
    path = base + "_debug" + extension

    # Finally, join the whole thing to the output directory.
    return os.path.join(output_dir, path)

def main(args=None):
    # parse arguments
    if args is None:
        args = sys.argv[1:]
    args = parse_args(args)

    # make sure keras is the minimum required version
    check_keras_version()

    # create the generator
    generator = create_generator(args)

    # optionally load config parameters
    if args.config:
        args.config = read_config_file(args.config)

    # optionally load anchor parameters
    anchor_params = None
    if args.config and 'anchor_parameters' in args.config:
        anchor_params = parse_anchor_parameters(args.config)

    # create the display window if necessary
    if not args.no_gui:
        cv2.namedWindow('Image', cv2.WINDOW_NORMAL)

    run(generator, args, anchor_params=anchor_params)

if __name__ == '__main__':
    main()
<|endoftext|>