<commit_before>#!/usr/bin/env python

"""
Python script to obtain GitHub Release download count and other statistics.
"""

import os
import sys
import json
import time

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

try:
    input = raw_input
except NameError:
    pass

__version__ = "1.2.0"
__author__ = "Alexander Gorishnyak"
__email__ = "kefir500@gmail.com"
__license__ = "MIT"

class ConnectionError(Exception):
    """
    Raised on connection error.
    """

    def __init__(self, reason=None):
        """
        :param reason: Connection error reason.
        """
        self.reason = reason if reason else "Unknown reason."

    def __str__(self):
        return str(self.reason)

class GithubError(Exception):
    """
    Generic exception raised on GitHub API HTTP error.
    """

    def __init__(self, code, message=None):
        """
        :param code: HTTP error code.
        :param message: Exception message text.
        """
        self.code = code
        self.message = message

    def __str__(self):
        return "GitHub API HTTP {0}{1}".format(self.code, (": " + self.message) if self.message else "")

class GithubRepoError(GithubError):
    """
    Raised when accessing nonexistent GitHub username, repository or release tag.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 404
        self.message = message if message else "Invalid GitHub username, repository or release tag."

class GithubLimitError(GithubError):
    """
    Raised when GitHub API request limit is exceeded.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 403
        self.message = message if message else "Request limit exceeded."

class GithubTokenError(GithubError):
    """
    Raised when trying to pass invalid GitHub OAuth token.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 401
        self.message = message if message else "Unauthorized. Check your \"GITHUB_TOKEN\" environment variable."

class _Text:
    """
    Definitions for colored output (no ANSI colors on Windows).
    """
    if os.name != "nt":
        HEADER = "\033[1;36m"
        SUCCESS = "\033[1;32m"
        ERROR = "\033[1;31m"
        BOLD = "\033[1m"
        ITALIC = "\033[3m"
        UNDERLINE = "\033[4m"
        END = "\033[0m"
    else:
        HEADER = SUCCESS = ERROR = BOLD = ITALIC = UNDERLINE = END = ""

def error(message):
    """
    Halt script due to critical error.
    :param message: Error text.
    """
    sys.exit(_Text.ERROR + "Error: " + message + _Text.END)

def get_env_token():
    """
    Get GitHub OAuth token from "GITHUB_TOKEN" environment variable.
    :return: GitHub OAuth token.
    """
    token = None if "GITHUB_TOKEN" not in os.environ else os.environ["GITHUB_TOKEN"]
    return token

def print_help():
    """
    Display command line help.
    """
    print("GitHub Download Stats")
    print("Python script to obtain GitHub Release download count and other statistics.\n")
    print("Usage:\n"
          "  python ghstats.py [{0}user{1}] [{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          "  python ghstats.py [{0}user{1}/{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          .format(_Text.ITALIC, _Text.END))
    print("Arguments:\n"
          "  {0}user{1}  Repository owner. If not present, user will be prompted for input.\n"
          "  {0}repo{1}  Repository title. If not present, user will be prompted for input.\n"
          "  {0}tag{1}   Release tag name. If not present, prints the total number of downloads.\n"
          .format(_Text.BOLD, _Text.END))
    print("Options:\n"
          "  {0}-d{1}, {0}--detail{1}  Print detailed statistics for release(s).\n"
          "  {0}-q{1}, {0}--quiet{1}   Print only resulting numbers and errors.\n"
          "                Overrides -d option.\n"
          "  {0}-l{1}, {0}--latest{1}  Get stats for the latest release.\n"
          "                Tag argument will be ignored.\n"
          "  {0}-h{1}, {0}--help{1}    Show this help.\n"
          .format(_Text.BOLD, _Text.END))
    print("Environment Variables:\n"
          "  {0}GITHUB_TOKEN{2}   GitHub OAuth token.\n"
          "                 Use to increase API request limit.\n"
          "                 {1}https://github.com/settings/tokens{2}"
          .format(_Text.BOLD, _Text.UNDERLINE, _Text.END))
    sys.exit(0)

def print_greeting():
    """
    Display greeting message.
    """
    print("GitHub Download Stats\n"
          "Author: Alexander Gorishnyak <kefir500@gmail.com>\n")

def download_stats(user=None, repo=None, tag=None, latest=False, token=None, quiet=False):
    """
    Get download statistics from GitHub API.
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Statistics on downloads.
    :raises GithubRepoError: When accessing nonexistent GitHub username, repository or release tag.
    :raises GithubLimitError: When GitHub API request limit is exceeded.
    :raises GithubTokenError: When trying to pass invalid GitHub API OAuth token.
    :raises ConnectionError: On connection error.
    """
    if not user:
        user = input("GitHub Username: ")
    if not repo:
        repo = input("GitHub Repository: ")
    if not quiet:
        print("Downloading {0}/{1} stats...".format(user, repo))
    url = "https://api.github.com/repos/{0}/{1}/releases".format(user, repo)
    url += ("" if not tag else "/tags/" + tag) if not latest else "/latest"
    headers = {} if not token else {"Authorization": "token " + token}
    request = urllib2.Request(url, headers=headers)
    start = time.time()
    try:
        response = urllib2.urlopen(request).read().decode("utf-8")
    except urllib2.HTTPError as e:
        if e.code == 404:
            raise GithubRepoError()    # Invalid GitHub username, repository or release tag.
        elif e.code == 403:
            raise GithubLimitError()   # GitHub API request limit exceeded.
        elif e.code == 401:
            raise GithubTokenError()   # Invalid GitHub OAuth token.
        else:
            raise GithubError(e.code)  # Generic GitHub API exception.
    except urllib2.URLError as e:
        raise ConnectionError(e.reason)
    stats = json.loads(response)
    if not quiet:
        end = time.time()
        print("Downloaded in {0:.3f}s".format(end - start))
    return stats

def get_release_downloads(release, quiet=False):
    """
    Get number of downloads for a single release.
    :param release: Release statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads for a release.
    """
    downloads_total = 0
    if not quiet:
        encoding = sys.stdout.encoding or "ascii"
        title = (release["name"] or release["tag_name"]).encode(encoding, 'ignore').decode(encoding)
        published = (time.strftime("%c", time.strptime(release["published_at"], "%Y-%m-%dT%H:%M:%SZ"))
                     if release["published_at"] else "Unpublished")
        print("")
        print("          --- " + _Text.HEADER + title + _Text.END + " ---\n")
        print("         Tag: " + release["tag_name"])
        print("      Author: " + release["author"]["login"])
        print("         URL: " + _Text.UNDERLINE + release["html_url"] + _Text.END)
        print("Published at: " + published)
        print("")
    if "assets" in release:
        indent = 13 + len(_Text.BOLD + _Text.END)
        for package in release["assets"]:
            downloads_release = package["download_count"]
            downloads_total += downloads_release
            if not quiet:
                print("{1:>{0}} {2}".format(
                      indent, _Text.BOLD + str(downloads_release) + _Text.END,
                      (package["label"] if package["label"] else package["name"])))
        if not quiet:
            print("{1:>{0}} Total".format(indent, _Text.BOLD + str(downloads_total) + _Text.END))
    return downloads_total

def get_stats_downloads(stats, quiet=False):
    """
    Get number of downloads from statistics.
    :param stats: Statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    total = 0
    if isinstance(stats, dict):
        total = get_release_downloads(stats, quiet)
    else:
        for release in stats:
            total += get_release_downloads(release, quiet)
    return total

def print_total(total, quiet=False, tag=None):
    """
    Print total number of downloads.
    :param total: Total number of downloads.
    :param quiet: If True, print only number of downloads without an additional text.
    :param tag: Release tag name (optional).
    :return: Total number of downloads passed via "total" parameter.
    """
    if not quiet:
        print("\n{0}Total Downloads{1}: {2}"
              .format(_Text.BOLD,
                      "" if not tag else " (" + tag + ")",
                      _Text.SUCCESS + str(total) + _Text.END))
    elif __name__ == "__main__" or __name__ == "ghstats.ghstats":  # pragma: no cover
        print(str(total))
    return total

def main(user=None, repo=None, tag=None, latest=False, detail=False, token=None, quiet=False):
    """
    Get number of downloads for GitHub release(s).
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param detail: Detailed output containing release information.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    if not quiet:
        print_greeting()
    try:
        stats = download_stats(user, repo, tag, latest, token, quiet)
    except GithubError as e:
        error(e.message)
    except ConnectionError as e:
        error(str(e))
    else:
        total = get_stats_downloads(stats, quiet or not detail)
        print_total(total, quiet, tag or (stats["tag_name"] if latest else None))
        return total

def main_cli(args=None):
    """
    Parse command line arguments and pass to the main function.
    :param args: Command line arguments (without script name).
    :return: Number of downloads.
    """
    user = None              # GitHub username
    repo = None              # GitHub repository
    tag = None               # GitHub release tag
    latest = False           # Latest release
    detail = False           # Detailed output
    quiet = False            # Quiet output
    token = get_env_token()  # GitHub token
    if args is None:
        args = sys.argv[1:]
    for arg in args:
        if arg == "-q" or arg == "--quiet":
            quiet = True
        elif arg == "-d" or arg == "--detail":
            detail = True
        elif arg == "-l" or arg == "--latest":
            latest = True
        elif arg == "-h" or arg == "--help" or arg == "-?":
            print_help()
        else:
            if not user:
                if "/" not in arg:
                    user = arg
                else:
                    userrepo = arg.split("/")
                    user = userrepo[0]
                    repo = userrepo[1]
            elif not repo:
                repo = arg
            elif quiet and detail and latest:
                break
            elif not tag:
                tag = arg
    return main(user, repo, tag, latest, detail, token, quiet)

if __name__ == "__main__":  # pragma: no cover
    try:
        main_cli(sys.argv[1:])
    except KeyboardInterrupt:
        pass
<commit_msg>Increase per-page argument to 100<commit_after>#!/usr/bin/env python

"""
Python script to obtain GitHub Release download count and other statistics.
"""

import os
import sys
import json
import time

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

try:
    input = raw_input
except NameError:
    pass

__version__ = "1.2.0"
__author__ = "Alexander Gorishnyak"
__email__ = "kefir500@gmail.com"
__license__ = "MIT"

class ConnectionError(Exception):
    """
    Raised on connection error.
    """

    def __init__(self, reason=None):
        """
        :param reason: Connection error reason.
        """
        self.reason = reason if reason else "Unknown reason."

    def __str__(self):
        return str(self.reason)

class GithubError(Exception):
    """
    Generic exception raised on GitHub API HTTP error.
    """

    def __init__(self, code, message=None):
        """
        :param code: HTTP error code.
        :param message: Exception message text.
        """
        self.code = code
        self.message = message

    def __str__(self):
        return "GitHub API HTTP {0}{1}".format(self.code, (": " + self.message) if self.message else "")

class GithubRepoError(GithubError):
    """
    Raised when accessing nonexistent GitHub username, repository or release tag.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 404
        self.message = message if message else "Invalid GitHub username, repository or release tag."

class GithubLimitError(GithubError):
    """
    Raised when GitHub API request limit is exceeded.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 403
        self.message = message if message else "Request limit exceeded."

class GithubTokenError(GithubError):
    """
    Raised when trying to pass invalid GitHub OAuth token.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 401
        self.message = message if message else "Unauthorized. Check your \"GITHUB_TOKEN\" environment variable."

class _Text:
    """
    Definitions for colored output (no ANSI colors on Windows).
    """
    if os.name != "nt":
        HEADER = "\033[1;36m"
        SUCCESS = "\033[1;32m"
        ERROR = "\033[1;31m"
        BOLD = "\033[1m"
        ITALIC = "\033[3m"
        UNDERLINE = "\033[4m"
        END = "\033[0m"
    else:
        HEADER = SUCCESS = ERROR = BOLD = ITALIC = UNDERLINE = END = ""

def error(message):
    """
    Halt script due to critical error.
    :param message: Error text.
    """
    sys.exit(_Text.ERROR + "Error: " + message + _Text.END)

def get_env_token():
    """
    Get GitHub OAuth token from "GITHUB_TOKEN" environment variable.
    :return: GitHub OAuth token.
    """
    token = None if "GITHUB_TOKEN" not in os.environ else os.environ["GITHUB_TOKEN"]
    return token

def print_help():
    """
    Display command line help.
    """
    print("GitHub Download Stats")
    print("Python script to obtain GitHub Release download count and other statistics.\n")
    print("Usage:\n"
          "  python ghstats.py [{0}user{1}] [{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          "  python ghstats.py [{0}user{1}/{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          .format(_Text.ITALIC, _Text.END))
    print("Arguments:\n"
          "  {0}user{1}  Repository owner. If not present, user will be prompted for input.\n"
          "  {0}repo{1}  Repository title. If not present, user will be prompted for input.\n"
          "  {0}tag{1}   Release tag name. If not present, prints the total number of downloads.\n"
          .format(_Text.BOLD, _Text.END))
    print("Options:\n"
          "  {0}-d{1}, {0}--detail{1}  Print detailed statistics for release(s).\n"
          "  {0}-q{1}, {0}--quiet{1}   Print only resulting numbers and errors.\n"
          "                Overrides -d option.\n"
          "  {0}-l{1}, {0}--latest{1}  Get stats for the latest release.\n"
          "                Tag argument will be ignored.\n"
          "  {0}-h{1}, {0}--help{1}    Show this help.\n"
          .format(_Text.BOLD, _Text.END))
    print("Environment Variables:\n"
          "  {0}GITHUB_TOKEN{2}   GitHub OAuth token.\n"
          "                 Use to increase API request limit.\n"
          "                 {1}https://github.com/settings/tokens{2}"
          .format(_Text.BOLD, _Text.UNDERLINE, _Text.END))
    sys.exit(0)

def print_greeting():
    """
    Display greeting message.
    """
    print("GitHub Download Stats\n"
          "Author: Alexander Gorishnyak <kefir500@gmail.com>\n")

def download_stats(user=None, repo=None, tag=None, latest=False, token=None, quiet=False):
    """
    Get download statistics from GitHub API.
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Statistics on downloads.
    :raises GithubRepoError: When accessing nonexistent GitHub username, repository or release tag.
    :raises GithubLimitError: When GitHub API request limit is exceeded.
    :raises GithubTokenError: When trying to pass invalid GitHub API OAuth token.
    :raises ConnectionError: On connection error.
    """
    if not user:
        user = input("GitHub Username: ")
    if not repo:
        repo = input("GitHub Repository: ")
    if not quiet:
        print("Downloading {0}/{1} stats...".format(user, repo))
    url = "https://api.github.com/repos/{0}/{1}/releases".format(user, repo)
    url += ("" if not tag else "/tags/" + tag) if not latest else "/latest"
    url += "?per_page=100"
    headers = {} if not token else {"Authorization": "token " + token}
    request = urllib2.Request(url, headers=headers)
    start = time.time()
    try:
        response = urllib2.urlopen(request).read().decode("utf-8")
    except urllib2.HTTPError as e:
        if e.code == 404:
            raise GithubRepoError()    # Invalid GitHub username, repository or release tag.
        elif e.code == 403:
            raise GithubLimitError()   # GitHub API request limit exceeded.
        elif e.code == 401:
            raise GithubTokenError()   # Invalid GitHub OAuth token.
        else:
            raise GithubError(e.code)  # Generic GitHub API exception.
    except urllib2.URLError as e:
        raise ConnectionError(e.reason)
    stats = json.loads(response)
    if not quiet:
        end = time.time()
        print("Downloaded in {0:.3f}s".format(end - start))
    return stats

def get_release_downloads(release, quiet=False):
    """
    Get number of downloads for a single release.
    :param release: Release statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads for a release.
    """
    downloads_total = 0
    if not quiet:
        encoding = sys.stdout.encoding or "ascii"
        title = (release["name"] or release["tag_name"]).encode(encoding, 'ignore').decode(encoding)
        published = (time.strftime("%c", time.strptime(release["published_at"], "%Y-%m-%dT%H:%M:%SZ"))
                     if release["published_at"] else "Unpublished")
        print("")
        print("          --- " + _Text.HEADER + title + _Text.END + " ---\n")
        print("         Tag: " + release["tag_name"])
        print("      Author: " + release["author"]["login"])
        print("         URL: " + _Text.UNDERLINE + release["html_url"] + _Text.END)
        print("Published at: " + published)
        print("")
    if "assets" in release:
        indent = 13 + len(_Text.BOLD + _Text.END)
        for package in release["assets"]:
            downloads_release = package["download_count"]
            downloads_total += downloads_release
            if not quiet:
                print("{1:>{0}} {2}".format(
                      indent, _Text.BOLD + str(downloads_release) + _Text.END,
                      (package["label"] if package["label"] else package["name"])))
        if not quiet:
            print("{1:>{0}} Total".format(indent, _Text.BOLD + str(downloads_total) + _Text.END))
    return downloads_total

def get_stats_downloads(stats, quiet=False):
    """
    Get number of downloads from statistics.
    :param stats: Statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    total = 0
    if isinstance(stats, dict):
        total = get_release_downloads(stats, quiet)
    else:
        for release in stats:
            total += get_release_downloads(release, quiet)
    return total

def print_total(total, quiet=False, tag=None):
    """
    Print total number of downloads.
    :param total: Total number of downloads.
    :param quiet: If True, print only number of downloads without an additional text.
    :param tag: Release tag name (optional).
    :return: Total number of downloads passed via "total" parameter.
    """
    if not quiet:
        print("\n{0}Total Downloads{1}: {2}"
              .format(_Text.BOLD,
                      "" if not tag else " (" + tag + ")",
                      _Text.SUCCESS + str(total) + _Text.END))
    elif __name__ == "__main__" or __name__ == "ghstats.ghstats":  # pragma: no cover
        print(str(total))
    return total

def main(user=None, repo=None, tag=None, latest=False, detail=False, token=None, quiet=False):
    """
    Get number of downloads for GitHub release(s).
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param detail: Detailed output containing release information.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    if not quiet:
        print_greeting()
    try:
        stats = download_stats(user, repo, tag, latest, token, quiet)
    except GithubError as e:
        error(e.message)
    except ConnectionError as e:
        error(str(e))
    else:
        total = get_stats_downloads(stats, quiet or not detail)
        print_total(total, quiet, tag or (stats["tag_name"] if latest else None))
        return total

def main_cli(args=None):
    """
    Parse command line arguments and pass to the main function.
    :param args: Command line arguments (without script name).
    :return: Number of downloads.
    """
    user = None              # GitHub username
    repo = None              # GitHub repository
    tag = None               # GitHub release tag
    latest = False           # Latest release
    detail = False           # Detailed output
    quiet = False            # Quiet output
    token = get_env_token()  # GitHub token
    if args is None:
        args = sys.argv[1:]
    for arg in args:
        if arg == "-q" or arg == "--quiet":
            quiet = True
        elif arg == "-d" or arg == "--detail":
            detail = True
        elif arg == "-l" or arg == "--latest":
            latest = True
        elif arg == "-h" or arg == "--help" or arg == "-?":
            print_help()
        else:
            if not user:
                if "/" not in arg:
                    user = arg
                else:
                    userrepo = arg.split("/")
                    user = userrepo[0]
                    repo = userrepo[1]
            elif not repo:
                repo = arg
            elif quiet and detail and latest:
                break
            elif not tag:
                tag = arg
    return main(user, repo, tag, latest, detail, token, quiet)

if __name__ == "__main__":  # pragma: no cover
    try:
        main_cli(sys.argv[1:])
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Azure Linux extension
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
# documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
# Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import datetime
import urllib2
import time
import traceback

def get_imds_data(node, json=True):
    """
    Query IMDS endpoint for instance metadata and return the response as a Json string.
    """
    if not node:
        return None
    if node[0] != '/':
        node = '/' + node
    imds_url = 'http://169.254.169.254{0}{1}'.format(node, '?format=json' if json else '')
    imds_headers = {'Metadata': 'True'}
    req = urllib2.Request(url=imds_url, headers=imds_headers)
    resp = urllib2.urlopen(req)
    data = resp.read()
    data_str = data.decode('utf-8')
    return data_str

class ImdsLogger:
    """
    Periodically probes IMDS endpoint and log the result as WALA events.
    """

    def __init__(self, ext_name, ext_ver, ext_op_type, ext_event_logger, ext_logger=None,
                 imds_data_getter=get_imds_data, logging_interval_in_minutes=60):
        self._ext_name = ext_name
        self._ext_ver = ext_ver
        self._ext_op_type = ext_op_type
        self._ext_logger = ext_logger  # E.g., hutil.log
        self._ext_event_logger = ext_event_logger  # E.g., waagent.AddExtensionEvent
        self._last_log_time = None
        self._imds_data_getter = imds_data_getter
        self._logging_interval = datetime.timedelta(minutes=logging_interval_in_minutes)

    def log_imds_data_if_right_time(self, log_as_ext_event=False):
        now = datetime.datetime.now()
        not_yet = self._last_log_time and now < self._last_log_time + self._logging_interval
        if not_yet:
            return

        try:
            imds_data = self._imds_data_getter('/metadata/latest/instance/')
        except Exception as e:
            if self._ext_logger:
                self._ext_logger('Exception occurred while getting IMDS data: {0}\n'
                                 'stacktrace: {1}').format(e, traceback.format_exc())
            imds_data = '{0}'.format(e)

        msg = 'IMDS instance data = {0}'.format(imds_data)
        if log_as_ext_event:
            self._ext_event_logger(name=self._ext_name,
                                   op=self._ext_op_type,
                                   isSuccess=True,
                                   version=self._ext_ver,
                                   message=msg)
        if self._ext_logger:
            self._ext_logger(msg)
        self._last_log_time = now

if __name__ == '__main__':

    def fake_get_imds_data(node, json=True):
        result = 'fake_get_imds_data(node="{0}", json="{1}")'.format(node, json)
        print result
        return result

    def default_ext_logger(msg):
        print 'default_ext_logger(msg="{0}")'.format(msg)

    def default_ext_event_logger(*args, **kwargs):
        print 'default_ext_event_logger(*args, **kwargs)'
        print 'args:'
        for arg in args:
            print arg
        print 'kwargs:'
        for k in kwargs:
            print('"{0}"="{1}"'.format(k, kwargs[k]))

    imds_logger = ImdsLogger('Microsoft.OSTCExtensions.LinuxDiagnostic', '2.3.9021', 'Heartbeat',
                             ext_logger=default_ext_logger, ext_event_logger=default_ext_event_logger,
                             imds_data_getter=fake_get_imds_data, logging_interval_in_minutes=1)
    start_time = datetime.datetime.now()
    done = False
    while not done:
        now = datetime.datetime.now()
        print 'Test loop iteration starting at {0}'.format(now)
        imds_logger.log_imds_data_if_right_time()
        if now >= start_time + datetime.timedelta(minutes=2):
            done = True
        else:
            print 'Sleeping 10 seconds'
            time.sleep(10)
<commit_msg>[LAD] Reflecting PR feedback on IMDS<commit_after>#!/usr/bin/env python
#
# Azure Linux extension
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
# documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
# Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import datetime
import urllib2
import time
import traceback

def get_imds_data(node, json=True):
    """
    Query IMDS endpoint for instance metadata and return the response as a Json string.

    :param str node: Instance metadata node we are querying about
    :param bool json: Indicates whether to query for Json output or not
    :return: Queried IMDS result in string
    :rtype: str
    """
    if not node:
        return None
    separator = '' if node[0] == '/' else '/'
    imds_url = 'http://169.254.169.254{0}{1}{2}'.format(separator, node, '?format=json' if json else '')
    imds_headers = {'Metadata': 'True'}
    req = urllib2.Request(url=imds_url, headers=imds_headers)
    resp = urllib2.urlopen(req)
    data = resp.read()
    data_str = data.decode('utf-8')
    return data_str

class ImdsLogger:
    """
    Periodically probes IMDS endpoint and log the result as WALA events.
    """

    def __init__(self, ext_name, ext_ver, ext_op_type, ext_event_logger, ext_logger=None,
                 imds_data_getter=get_imds_data, logging_interval_in_minutes=60):
        """
        Constructor
        :param str ext_name: Extension name (e.g., hutil.get_name())
        :param str ext_ver: Extension version (e.g., hutil.get_version())
        :param str ext_op_type: Extension operation type (e.g., HeartBeat)
        :param ext_event_logger: Extension event logger (e.g., waagent.AddExtensionEvent)
        :param ext_logger: Extension message logger (e.g., hutil.log)
        :param imds_data_getter: IMDS data getter function (e.g., get_imds_data)
        :param int logging_interval_in_minutes: Logging interval in minutes
        """
        self._ext_name = ext_name
        self._ext_ver = ext_ver
        self._ext_op_type = ext_op_type
        self._ext_logger = ext_logger  # E.g., hutil.log
        self._ext_event_logger = ext_event_logger  # E.g., waagent.AddExtensionEvent
        self._last_log_time = datetime.datetime.fromordinal(1)
        self._imds_data_getter = imds_data_getter
        self._logging_interval = datetime.timedelta(minutes=logging_interval_in_minutes)

    def _ext_log_if_enabled(self, msg):
        """
        Log an extension message if logger is specified.
        :param str msg: Message to log
        :return: None
        """
        if self._ext_logger:
            self._ext_logger(msg)

    def log_imds_data_if_right_time(self, log_as_ext_event=False):
        """
        Query and log IMDS data if it's right time to do so.
        :param bool log_as_ext_event: Indicates whether to log IMDS data as a waagent/extension event.
        :return: None
        """
        now = datetime.datetime.now()
        if now < self._last_log_time + self._logging_interval:
            return

        try:
            imds_data = self._imds_data_getter('/metadata/latest/instance/')
        except Exception as e:
            self._ext_log_if_enabled('Exception occurred while getting IMDS data: {0}\n'
                                     'stacktrace: {1}').format(e, traceback.format_exc())
            imds_data = '{0}'.format(e)

        msg = 'IMDS instance data = {0}'.format(imds_data)
        if log_as_ext_event:
            self._ext_event_logger(name=self._ext_name,
                                   op=self._ext_op_type,
                                   isSuccess=True,
                                   version=self._ext_ver,
                                   message=msg)
        self._ext_log_if_enabled(msg)
        self._last_log_time = now

if __name__ == '__main__':

    def fake_get_imds_data(node, json=True):
        result = 'fake_get_imds_data(node="{0}", json="{1}")'.format(node, json)
        print result
        return result

    def default_ext_logger(msg):
        print 'default_ext_logger(msg="{0}")'.format(msg)

    def default_ext_event_logger(*args, **kwargs):
        print 'default_ext_event_logger(*args, **kwargs)'
        print 'args:'
        for arg in args:
            print arg
        print 'kwargs:'
        for k in kwargs:
            print('"{0}"="{1}"'.format(k, kwargs[k]))

    imds_logger = ImdsLogger('Microsoft.OSTCExtensions.LinuxDiagnostic', '2.3.9021', 'Heartbeat',
                             ext_logger=default_ext_logger, ext_event_logger=default_ext_event_logger,
                             imds_data_getter=fake_get_imds_data, logging_interval_in_minutes=1)
    start_time = datetime.datetime.now()
    done = False
    while not done:
        now = datetime.datetime.now()
        print 'Test loop iteration starting at {0}'.format(now)
        imds_logger.log_imds_data_if_right_time()
        if now >= start_time + datetime.timedelta(minutes=2):
            done = True
        else:
            print 'Sleeping 10 seconds'
            time.sleep(10)
<|endoftext|>


<commit_before>from django.db import transaction
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from django_filters.rest_framework import FilterSet
from rest_framework.exceptions import APIException
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet
from rest_framework.serializers import ValidationError

from contentcuration.models import ContentNode

class MissingRequiredParamsException(APIException):
    status_code = 412
    default_detail = "Required query parameters were missing from the request"
    default_code = "missing_parameters"

_valid_positions = set(["first-child", "last-child", "left", "right",])

class TreeFilter(FilterSet):
    class Meta:
        model = ContentNode
        fields = ("parent",)

class TreeViewSet(GenericViewSet):
    filter_backends = (DjangoFilterBackend,)
    filter_class = TreeFilter
    values = (
        "id",
        "tree_id",
        "lft",
        "parent",
    )

    field_map = {
        "sort_order": "lft",
    }

    def list(self, request, *args, **kwargs):
        channel_id = request.query_params.get("channel_id")
        if channel_id is None:
            raise MissingRequiredParamsException(
                "channel_id query parameter is required but was missing from the request"
            )
        root = get_object_or_404(ContentNode, channel_main=channel_id)

        def map_data(item):
            item["sort_order"] = item.pop("lft")
            item["channel_id"] = channel_id
            return item

        queryset = self.filter_queryset(root.get_descendants())
        tree = map(map_data, queryset.values(*self.values))
        return Response(tree)

    def move(self, pk, *args, **kwargs):
        contentnode = get_object_or_404(ContentNode, pk=pk)
        target = kwargs.pop("target", None)
        try:
            if target is None:
                raise ValidationError("A target content node must be specified")
            try:
                target = ContentNode.get(pk=target)
            except ContentNode.DoesNotExist:
                raise ValidationError(
                    "Target content node: {} does not exist".format(target)
                )
            except ValueError:
                raise ValidationError(
                    "Invalid target content node specified: {}".format(target)
                )
            position = kwargs.pop("position", "first-child")
            if position not in _valid_positions:
                raise ValidationError(
                    "Invalid node position specified, must be one of {}".format(
                        ", ".join(_valid_positions)
                    )
                )
            try:
                with transaction.atomic():
                    # Lock only MPTT columns for updates on this tree and the target tree
                    # until the end of this transaction
                    ContentNode.objects.select_for_update().order_by().filter(
                        Q(tree_id=contentnode.tree_id) | Q(tree_id=target.tree_id)
                    ).values("tree_id", "lft", "rght")
                    contentnode.move_to(target, position)

            except ValueError:
                raise ValidationError(
                    "Invalid position argument specified: {}".format(position)
                )
            return None, None
        except ValidationError as e:
            return None, str(e)
<commit_msg>Reduce complexity of tree move function.<commit_after>from django.db import transaction
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from django_filters.rest_framework import FilterSet
from rest_framework.exceptions import APIException
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet
from rest_framework.serializers import ValidationError

from contentcuration.models import ContentNode

class MissingRequiredParamsException(APIException):
    status_code = 412
    default_detail = "Required query parameters were missing from the request"
    default_code = "missing_parameters"

_valid_positions = set(["first-child", "last-child", "left", "right"])

class TreeFilter(FilterSet):
    class Meta:
        model = ContentNode
        fields = ("parent",)

def validate_move_args(target, position):
    if target is None:
        raise ValidationError("A target content node must be specified")
    try:
        target = ContentNode.get(pk=target)
    except ContentNode.DoesNotExist:
        raise ValidationError("Target content node: {} does not exist".format(target))
    except ValueError:
        raise ValidationError(
            "Invalid target content node specified: {}".format(target)
        )
    if position not in _valid_positions:
        raise ValidationError(
            "Invalid node position specified, must be one of {}".format(
                ", ".join(_valid_positions)
            )
        )
    return target, position

class TreeViewSet(GenericViewSet):
    filter_backends = (DjangoFilterBackend,)
    filter_class = TreeFilter
    values = (
        "id",
        "tree_id",
        "lft",
        "parent",
    )

    field_map = {
        "sort_order": "lft",
    }

    def list(self, request, *args, **kwargs):
        channel_id = request.query_params.get("channel_id")
        if channel_id is None:
            raise MissingRequiredParamsException(
                "channel_id query parameter is required but was missing from the request"
            )
        root = get_object_or_404(ContentNode, channel_main=channel_id)

        def map_data(item):
            item["sort_order"] = item.pop("lft")
            item["channel_id"] = channel_id
            return item

        queryset = self.filter_queryset(root.get_descendants())
        tree = map(map_data, queryset.values(*self.values))
        return Response(tree)

    def move(self, pk, *args, **kwargs):
        contentnode = get_object_or_404(ContentNode, pk=pk)
        target = kwargs.pop("target", None)
        position = kwargs.pop("position", "first-child")
        try:
            target, position = validate_move_args(target, position)
            try:
                with transaction.atomic():
                    # Lock only MPTT columns for updates on this tree and the target tree
                    # until the end of this transaction
                    ContentNode.objects.select_for_update().order_by().filter(
                        Q(tree_id=contentnode.tree_id) | Q(tree_id=target.tree_id)
                    ).values("tree_id", "lft", "rght")
                    contentnode.move_to(target, position)

            except ValueError:
                raise ValidationError(
                    "Invalid position argument specified: {}".format(position)
                )
            return None, None
        except ValidationError as e:
            return None, str(e)
<|endoftext|>


<commit_before>import logging
logger = logging.getLogger(__name__)

import os
import shutil

import click

from fuzzywuzzy import process

from mediasort.act import Action
from mediasort.lib import as_list
from mediasort.lib import validate

class CopyTo(Action):
    __key__ = "copy_to"

    def validate_options(self):
        self.dir_must_exist(self.destination(), 'destination')

    def destination(self):
        return self.options.get('destination')

    def perform(self, dry_run):
        path = self.options['target'].path
        self.copy_file_or_dir(path, dry_run=dry_run)

    def copy_file_or_dir(self, path, dry_run):
        logger.info("copying '%s' to '%s'" % (path, self.destination()))
        if os.path.isfile(path):
            action = self.copy_file
        elif os.path.isdir(path):
            action = self.copy_dir
        else:
            raise click.ClickException("Couldn't find path: %s" % path)

        if not dry_run:
            action(path, self.destination())

    def copy_file(self, src, dst_root):
        shutil.copy(src, dst_root)

    def copy_dir(self, src, dst_root):
        dst = os.path.join(dst_root, os.path.basename(src))
        shutil.copytree(src, dst)
        self.set_mode(dst, self.options.get('permissions'))

    @classmethod
    def set_mode(cls, path, mode):
        if mode:
            logger.debug("set '%s' to '%r'" % (path, mode))
            os.chmod(dst, mode)
        else:
            logger.debug("'%s': mode was empty" % path)

class CopyToSubdir(CopyTo):
    __key__ = 'copy_to_subdir'

    def validate_options(self):
        self.dir_must_exist(self.options.get('destination'), 'destination')
        self.validate_subdir()
        self.ensure_destination_exists()

    def ensure_destination_exists(self):
        path = self.destination()
        if not os.path.exists(path):
            os.makedirs(path)
            self.set_mode(dst, self.options.get('permissions'))

    def validate_subdir(self):
        if 'subdir' not in self.options:
            raise click.ClickException("copy_to_subdir needs a subdir")

    def subdir():
        return self.options.get('subdir')

    def destination(self):
        return os.path.join(*filter(None, [
            self.options.get('destination'),
            self.subdir(),
        ]))

class CopyToMatchingSubdir(CopyToSubdir):
    __key__ = 'copy_to_matching_subdir'

    def validate_subdir(self):
        if 'subdir' in self.options:
            raise click.ClickException(
                "can't pass a subdir to copy_to_matching_subdir"
            )

    def subdir(self):
        if 'chosen_subdir' not in self.options:
            self.choose_subdir()
        return self.options['chosen_subdir']

    def choose_subdir(self):
        root = self.options.get('destination')
        haystack = [d for d in os.listdir(root)
                    if os.path.isdir(os.path.join(root, d))]

        name = os.path.basename(self.options['target'].path)
        choice = process.extractOne(name, haystack)

        if choice[1] >= self.options.get('score_cutoff', 90):
            self.options['chosen_subdir'] = choice[0]
        else:
            self.options['chosen_subdir'] = self.options['target'].name

class CopyContentsTo(CopyToSubdir):
    __key__ = 'copy_contents_to'

    def subdir(self):
        if self.options.get('preserve_dir', False):
            return os.path.basename(self.options['target'].path)

        return None

    def validate_subdir(self): pass

    def perform(self, dry_run):
        path = self.options['target'].path
        contents = [os.path.join(path, f) for f in os.listdir(path)]
        only = as_list(self.options.get('only', []))
        exclude = as_list(self.options.get('exclude', []))

        if only:
            logger.debug("Only: %s" % only)
        if exclude:
            logger.debug("Exclude: %s" % exclude)

        for f in contents:
            ext = os.path.splitext(f)[1]
            if ext.startswith('.'):
                ext = ext[1:]

            if ((only and ext in only) or
              (not only and exclude and ext not in exclude) or
              (not only and not exclude)):
                self.copy_file_or_dir(f, dry_run=dry_run)

            else:
                logger.debug("%s did not pass extension filter" % f)

<commit_msg>removed extra debug line<commit_after>import logging
logger = logging.getLogger(__name__)

import os
import shutil

import click

from fuzzywuzzy import process

from mediasort.act import Action
from mediasort.lib import as_list
from mediasort.lib import validate

class CopyTo(Action):
    __key__ = "copy_to"

    def validate_options(self):
        self.dir_must_exist(self.destination(), 'destination')

    def destination(self):
        return self.options.get('destination')

    def perform(self, dry_run):
        path = self.options['target'].path
        self.copy_file_or_dir(path, dry_run=dry_run)

    def copy_file_or_dir(self, path, dry_run):
        logger.info("copying '%s' to '%s'" % (path, self.destination()))
        if os.path.isfile(path):
            action = self.copy_file
        elif os.path.isdir(path):
            action = self.copy_dir
        else:
            raise click.ClickException("Couldn't find path: %s" % path)

        if not dry_run:
            action(path, self.destination())

    def copy_file(self, src, dst_root):
        shutil.copy(src, dst_root)

    def copy_dir(self, src, dst_root):
        dst = os.path.join(dst_root, os.path.basename(src))
        shutil.copytree(src, dst)
        self.set_mode(dst, self.options.get('permissions'))

    @classmethod
    def set_mode(cls, path, mode):
        if mode:
            logger.debug("set '%s' to '%r'" % (path, mode))
            os.chmod(dst, mode)

class CopyToSubdir(CopyTo):
    __key__ = 'copy_to_subdir'

    def validate_options(self):
        self.dir_must_exist(self.options.get('destination'), 'destination')
        self.validate_subdir()
        self.ensure_destination_exists()

    def ensure_destination_exists(self):
        path = self.destination()
        if not os.path.exists(path):
            os.makedirs(path)
            self.set_mode(dst, self.options.get('permissions'))

    def validate_subdir(self):
        if 'subdir' not in self.options:
            raise click.ClickException("copy_to_subdir needs a subdir")

    def subdir():
        return self.options.get('subdir')

    def destination(self):
        return os.path.join(*filter(None, [
            self.options.get('destination'),
            self.subdir(),
        ]))

class CopyToMatchingSubdir(CopyToSubdir):
    __key__ = 'copy_to_matching_subdir'

    def validate_subdir(self):
        if 'subdir' in self.options:
            raise click.ClickException(
                "can't pass a subdir to copy_to_matching_subdir"
            )

    def subdir(self):
        if 'chosen_subdir' not in self.options:
            self.choose_subdir()
        return self.options['chosen_subdir']

    def choose_subdir(self):
        root = self.options.get('destination')
        haystack = [d for d in os.listdir(root)
                    if os.path.isdir(os.path.join(root, d))]

        name = os.path.basename(self.options['target'].path)
        choice = process.extractOne(name, haystack)

        if choice[1] >= self.options.get('score_cutoff', 90):
            self.options['chosen_subdir'] = choice[0]
        else:
            self.options['chosen_subdir'] = self.options['target'].name

class CopyContentsTo(CopyToSubdir):
    __key__ = 'copy_contents_to'

    def subdir(self):
        if self.options.get('preserve_dir', False):
            return os.path.basename(self.options['target'].path)

        return None

    def validate_subdir(self): pass

    def perform(self, dry_run):
        path = self.options['target'].path
        contents = [os.path.join(path, f) for f in os.listdir(path)]
        only = as_list(self.options.get('only', []))
        exclude = as_list(self.options.get('exclude', []))

        if only:
            logger.debug("Only: %s" % only)
        if exclude:
            logger.debug("Exclude: %s" % exclude)

        for f in contents:
            ext = os.path.splitext(f)[1]
            if ext.startswith('.'):
                ext = ext[1:]

            if ((only and ext in only) or
              (not only and exclude and ext not in exclude) or
              (not only and not exclude)):
                self.copy_file_or_dir(f, dry_run=dry_run)

            else:
                logger.debug("%s did not pass extension filter" % f)

<|endoftext|>


<commit_before>import json
import io
from collections import namedtuple
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpRequest
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.template.loader import render_to_string
from django.core.urlresolvers import reverse
from django.contrib.messages.api import get_messages
from django.test.client import encode_multipart

__version__ = '0.1.0'
Response = namedtuple('Response',
                      'status_code content location messages headers')

class ViewTestCase:
    def setUp(self):
        super().setUp()

        if hasattr(self, 'setup_models'):
            self.setup_models()

    def request(self, method='GET', user=AnonymousUser(), url_kwargs={},
                post_data={}, get_data={}, view_kwargs={}):
        self._request = HttpRequest()
        setattr(self._request, 'method', method)
        setattr(self._request, 'user', user)

        setattr(self._request, 'session', 'session')
        self.messages = FallbackStorage(self._request)
        setattr(self._request, '_messages', self.messages)

        setattr(self._request, 'GET', self._get_get_data(get_data))

        if method in ['POST', 'PATCH', 'PUT']:
            post_data = self._get_post_data(post_data)
            setattr(self._request, method, post_data)

        url_params = self._get_url_kwargs(url_kwargs)
        view = self.setup_view(view_kwargs=view_kwargs)

        response = view(self._request, **url_params)

        content = None
        if hasattr(response, 'render'):
            content = response.render().content.decode('utf-8')
        elif hasattr(response, 'content'):
            content = response.content.decode('utf-8')

        return Response(
            status_code=response.status_code,
            content=content,
            location=response.get('location', None),
            messages=[str(m) for m in get_messages(self._request)],
            headers=response._headers
        )

    def setup_view(self, view_kwargs={}):
        if not hasattr(self, 'view_class'):
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'view_class' or an implementation of 'setup_view()'")

        if hasattr(self, 'viewset_actions'):
            return self.view_class.as_view(self.viewset_actions, **view_kwargs)
        else:
            return self.view_class.as_view(**view_kwargs)

    def _get_get_data(self, data={}):
        get_data = {}
        if hasattr(self, 'setup_get_data'):
            get_data = self.setup_get_data()
        elif hasattr(self, 'get_data'):
            get_data = self.get_data.copy()

        get_data.update(data)
        return get_data

    def _get_post_data(self, data={}):
        post_data = {}
        if hasattr(self, 'setup_post_data'):
            post_data = self.setup_post_data()
        elif hasattr(self, 'post_data'):
            post_data = self.post_data.copy()

        post_data.update(data)
        return post_data

    def _get_url_kwargs(self, add_args={}):
        url_kwargs = {}
        if hasattr(self, 'setup_url_kwargs'):
            url_kwargs = self.setup_url_kwargs()
        elif hasattr(self, 'url_kwargs'):
            url_kwargs = self.url_kwargs.copy()

        url_kwargs.update(add_args)
        return url_kwargs

    def _get_template(self):
        if hasattr(self, 'setup_template'):
            return self.setup_template()
        elif hasattr(self, 'template'):
            return self.template
        else:
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'template' or an implementation of 'setup_template()'")

    def _get_template_context(self):
        if hasattr(self, 'setup_template_context'):
            return self.setup_template_context()
        elif hasattr(self, 'template_context'):
            return self.template_context
        else:
            return {}

    def render_content(self, **context_kwargs):
        template = self._get_template()
        context = self._get_template_context()
        context.update(context_kwargs)

        return render_to_string(template, context, request=self._request)

    @property
    def expected_content(self):
        return self.render_content()

    def _get_success_url_kwargs(self):
        if hasattr(self, 'setup_success_url_kwargs'):
            return self.setup_success_url_kwargs()
        elif hasattr(self, 'success_url_kwargs'):
            return self.success_url_kwargs
        else:
            return self._get_url_kwargs()

    def get_success_url(self):
        if hasattr(self, 'success_url'):
            return self.success_url
        elif hasattr(self, 'success_url_name'):
            url_kwargs = self._get_success_url_kwargs()

            return reverse(self.success_url_name, kwargs=url_kwargs)
        else:
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'success_url', 'success_url_name' or an implementation of "
                "'get_success_url()'")

    @property
    def expected_success_url(self):
        return self.get_success_url()

class APITestCase(ViewTestCase):
    def request(self, method='GET', user=AnonymousUser(), url_kwargs={},
                post_data={}, get_data={}, content_type='application/json'):
        self._request = HttpRequest()
        setattr(self._request, 'method', method)
        setattr(self._request, '_force_auth_user', user)
        self._request.META['SERVER_NAME'] = 'testserver'
        self._request.META['SERVER_PORT'] = '80'

        url_params = self._get_url_kwargs(url_kwargs)
        view = self.setup_view()

        setattr(self._request, 'GET', self._get_get_data(get_data))

        if method in ['POST', 'PATCH', 'PUT']:
            post_data = self._get_post_data(post_data)

            if content_type == 'multipart/form-data':
                content_type = 'multipart/form-data; boundary=BoUnDaRyStRiNg'
                post_data = encode_multipart('BoUnDaRyStRiNg', post_data)

            else:
                post_data = json.dumps(post_data).encode()

            self._request.META['CONTENT_LENGTH'] = len(post_data)
            self._request.META['CONTENT_TYPE'] = content_type
            setattr(self._request, '_stream', io.BytesIO(post_data))

        response = view(self._request, **url_params)
        content = response.render().content.decode('utf-8')
        if 'application/json' in response._headers.get('content-type', ()):
            content = json.loads(content)
        elif 'application/xml' in response._headers.get('content-type', ()):
            content = response.data

        return Response(
            status_code=response.status_code,
            content=content,
            location=response.get('location', None),
            messages=[str(m) for m in get_messages(self._request)],
            headers=response._headers
        )
<commit_msg>Use real SessionStore<commit_after>import json
import io
from importlib import import_module
from collections import namedtuple
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpRequest
from django.contrib.auth.models import AnonymousUser
from django.contrib.messages.storage.fallback import FallbackStorage
from django.template.loader import render_to_string
from django.core.urlresolvers import reverse
from django.contrib.messages.api import get_messages
from django.test.client import encode_multipart
from django.conf import settings

__version__ = '0.1.0'
SessionStore = import_module(settings.SESSION_ENGINE).SessionStore
Response = namedtuple('Response',
                      'status_code content location messages headers')

class ViewTestCase:
    def setUp(self):
        super().setUp()

        if hasattr(self, 'setup_models'):
            self.setup_models()

    def request(self, method='GET', user=AnonymousUser(), url_kwargs={},
                post_data={}, get_data={}, view_kwargs={}):
        self._request = HttpRequest()
        setattr(self._request, 'method', method)
        setattr(self._request, 'user', user)

        setattr(self._request, 'session', SessionStore())
        self.messages = FallbackStorage(self._request)
        setattr(self._request, '_messages', self.messages)

        setattr(self._request, 'GET', self._get_get_data(get_data))

        if method in ['POST', 'PATCH', 'PUT']:
            post_data = self._get_post_data(post_data)
            setattr(self._request, method, post_data)

        url_params = self._get_url_kwargs(url_kwargs)
        view = self.setup_view(view_kwargs=view_kwargs)

        response = view(self._request, **url_params)

        content = None
        if hasattr(response, 'render'):
            content = response.render().content.decode('utf-8')
        elif hasattr(response, 'content'):
            content = response.content.decode('utf-8')

        return Response(
            status_code=response.status_code,
            content=content,
            location=response.get('location', None),
            messages=[str(m) for m in get_messages(self._request)],
            headers=response._headers
        )

    def setup_view(self, view_kwargs={}):
        if not hasattr(self, 'view_class'):
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'view_class' or an implementation of 'setup_view()'")

        if hasattr(self, 'viewset_actions'):
            return self.view_class.as_view(self.viewset_actions, **view_kwargs)
        else:
            return self.view_class.as_view(**view_kwargs)

    def _get_get_data(self, data={}):
        get_data = {}
        if hasattr(self, 'setup_get_data'):
            get_data = self.setup_get_data()
        elif hasattr(self, 'get_data'):
            get_data = self.get_data.copy()

        get_data.update(data)
        return get_data

    def _get_post_data(self, data={}):
        post_data = {}
        if hasattr(self, 'setup_post_data'):
            post_data = self.setup_post_data()
        elif hasattr(self, 'post_data'):
            post_data = self.post_data.copy()

        post_data.update(data)
        return post_data

    def _get_url_kwargs(self, add_args={}):
        url_kwargs = {}
        if hasattr(self, 'setup_url_kwargs'):
            url_kwargs = self.setup_url_kwargs()
        elif hasattr(self, 'url_kwargs'):
            url_kwargs = self.url_kwargs.copy()

        url_kwargs.update(add_args)
        return url_kwargs

    def _get_template(self):
        if hasattr(self, 'setup_template'):
            return self.setup_template()
        elif hasattr(self, 'template'):
            return self.template
        else:
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'template' or an implementation of 'setup_template()'")

    def _get_template_context(self):
        if hasattr(self, 'setup_template_context'):
            return self.setup_template_context()
        elif hasattr(self, 'template_context'):
            return self.template_context
        else:
            return {}

    def render_content(self, **context_kwargs):
        template = self._get_template()
        context = self._get_template_context()
        context.update(context_kwargs)

        return render_to_string(template, context, request=self._request)

    @property
    def expected_content(self):
        return self.render_content()

    def _get_success_url_kwargs(self):
        if hasattr(self, 'setup_success_url_kwargs'):
            return self.setup_success_url_kwargs()
        elif hasattr(self, 'success_url_kwargs'):
            return self.success_url_kwargs
        else:
            return self._get_url_kwargs()

    def get_success_url(self):
        if hasattr(self, 'success_url'):
            return self.success_url
        elif hasattr(self, 'success_url_name'):
            url_kwargs = self._get_success_url_kwargs()

            return reverse(self.success_url_name, kwargs=url_kwargs)
        else:
            raise ImproperlyConfigured(
                "ViewTestCase requires either a definition of "
                "'success_url', 'success_url_name' or an implementation of "
                "'get_success_url()'")

    @property
    def expected_success_url(self):
        return self.get_success_url()

class APITestCase(ViewTestCase):
    def request(self, method='GET', user=AnonymousUser(), url_kwargs={},
                post_data={}, get_data={}, content_type='application/json'):
        self._request = HttpRequest()
        setattr(self._request, 'method', method)
        setattr(self._request, '_force_auth_user', user)
        self._request.META['SERVER_NAME'] = 'testserver'
        self._request.META['SERVER_PORT'] = '80'

        url_params = self._get_url_kwargs(url_kwargs)
        view = self.setup_view()

        setattr(self._request, 'GET', self._get_get_data(get_data))

        if method in ['POST', 'PATCH', 'PUT']:
            post_data = self._get_post_data(post_data)

            if content_type == 'multipart/form-data':
                content_type = 'multipart/form-data; boundary=BoUnDaRyStRiNg'
                post_data = encode_multipart('BoUnDaRyStRiNg', post_data)

            else:
                post_data = json.dumps(post_data).encode()

            self._request.META['CONTENT_LENGTH'] = len(post_data)
            self._request.META['CONTENT_TYPE'] = content_type
            setattr(self._request, '_stream', io.BytesIO(post_data))

        response = view(self._request, **url_params)
        content = response.render().content.decode('utf-8')
        if 'application/json' in response._headers.get('content-type', ()):
            content = json.loads(content)
        elif 'application/xml' in response._headers.get('content-type', ()):
            content = response.data

        return Response(
            status_code=response.status_code,
            content=content,
            location=response.get('location', None),
            messages=[str(m) for m in get_messages(self._request)],
            headers=response._headers
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function
import unittest
import random
import time
import math
import sys
import array
import tarfile
import hashlib
import os
import getopt
import operator
import functools
import numpy as np
import cv2

# Python 3 moved urlopen to urllib.requests
try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen

class NewOpenCVTests(unittest.TestCase):

    def get_sample(self, filename, iscolor = cv2.IMREAD_COLOR):
        if not filename in self.image_cache:
            filedata = urlopen("https://raw.github.com/Itseez/opencv/master/" + filename).read()
            self.image_cache[filename] = cv2.imdecode(np.fromstring(filedata, dtype=np.uint8), iscolor)
        return self.image_cache[filename]

    def setUp(self):
        self.image_cache = {}

    def hashimg(self, im):
        """ Compute a hash for an image, useful for image comparisons """
        return hashlib.md5(im.tostring()).digest()

    if sys.version_info[:2] == (2, 6):
        def assertLess(self, a, b, msg=None):
            if not a < b:
                self.fail('%s not less than %s' % (repr(a), repr(b)))

        def assertLessEqual(self, a, b, msg=None):
            if not a <= b:
                self.fail('%s not less than or equal to %s' % (repr(a), repr(b)))

        def assertGreater(self, a, b, msg=None):
            if not a > b:
                self.fail('%s not greater than %s' % (repr(a), repr(b)))

# Tests to run first; check the handful of basic operations that the later tests rely on

class Hackathon244Tests(NewOpenCVTests):

    def test_int_array(self):
        a = np.array([-1, 2, -3, 4, -5])
        absa0 = np.abs(a)
        self.assertTrue(cv2.norm(a, cv2.NORM_L1) == 15)
        absa1 = cv2.absdiff(a, 0)
        self.assertEqual(cv2.norm(absa1, absa0, cv2.NORM_INF), 0)

    def test_imencode(self):
        a = np.zeros((480, 640), dtype=np.uint8)
        flag, ajpg = cv2.imencode("img_q90.jpg", a, [cv2.IMWRITE_JPEG_QUALITY, 90])
        self.assertEqual(flag, True)
        self.assertEqual(ajpg.dtype, np.uint8)
        self.assertGreater(ajpg.shape[0], 1)
        self.assertEqual(ajpg.shape[1], 1)

    def test_projectPoints(self):
        objpt = np.float64([[1,2,3]])
        imgpt0, jac0 = cv2.projectPoints(objpt, np.zeros(3), np.zeros(3), np.eye(3), np.float64([]))
        imgpt1, jac1 = cv2.projectPoints(objpt, np.zeros(3), np.zeros(3), np.eye(3), None)
        self.assertEqual(imgpt0.shape, (objpt.shape[0], 1, 2))
        self.assertEqual(imgpt1.shape, imgpt0.shape)
        self.assertEqual(jac0.shape, jac1.shape)
        self.assertEqual(jac0.shape[0], 2*objpt.shape[0])

    def test_estimateAffine3D(self):
        pattern_size = (11, 8)
        pattern_points = np.zeros((np.prod(pattern_size), 3), np.float32)
        pattern_points[:,:2] = np.indices(pattern_size).T.reshape(-1, 2)
        pattern_points *= 10
        (retval, out, inliers) = cv2.estimateAffine3D(pattern_points, pattern_points)
        self.assertEqual(retval, 1)
        if cv2.norm(out[2,:]) < 1e-3:
            out[2,2]=1
        self.assertLess(cv2.norm(out, np.float64([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]])), 1e-3)
        self.assertEqual(cv2.countNonZero(inliers), pattern_size[0]*pattern_size[1])

    def test_fast(self):
        fd = cv2.FastFeatureDetector_create(30, True)
        img = self.get_sample("samples/data/right02.jpg", 0)
        img = cv2.medianBlur(img, 3)
        imgc = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
        keypoints = fd.detect(img)
        self.assertTrue(600 <= len(keypoints) <= 700)
        for kpt in keypoints:
            self.assertNotEqual(kpt.response, 0)

    def check_close_angles(self, a, b, angle_delta):
        self.assertTrue(abs(a - b) <= angle_delta or
                        abs(360 - abs(a - b)) <= angle_delta)

    def check_close_pairs(self, a, b, delta):
        self.assertLessEqual(abs(a[0] - b[0]), delta)
        self.assertLessEqual(abs(a[1] - b[1]), delta)

    def check_close_boxes(self, a, b, delta, angle_delta):
        self.check_close_pairs(a[0], b[0], delta)
        self.check_close_pairs(a[1], b[1], delta)
        self.check_close_angles(a[2], b[2], angle_delta)

    def test_geometry(self):
        npt = 100
        np.random.seed(244)
        a = np.random.randn(npt,2).astype('float32')*50 + 150

        img = np.zeros((300, 300, 3), dtype='uint8')
        be = cv2.fitEllipse(a)
        br = cv2.minAreaRect(a)
        mc, mr = cv2.minEnclosingCircle(a)

        be0 = ((150.2511749267578, 150.77322387695312), (158.024658203125, 197.57696533203125), 37.57804489135742)
        br0 = ((161.2974090576172, 154.41793823242188), (199.2301483154297, 207.7177734375), -9.164555549621582)
        mc0, mr0 = (160.41790771484375, 144.55152893066406), 136.713500977

        self.check_close_boxes(be, be0, 5, 15)
        self.check_close_boxes(br, br0, 5, 15)
        self.check_close_pairs(mc, mc0, 5)
        self.assertLessEqual(abs(mr - mr0), 5)

if __name__ == '__main__':
    print("Testing OpenCV", cv2.__version__)
    random.seed(0)
    unittest.main()
<commit_msg>Python test: added options to provide local data path<commit_after>#!/usr/bin/env python

from __future__ import print_function
import unittest
import random
import time
import math
import sys
import array
import tarfile
import hashlib
import os
import getopt
import operator
import functools
import numpy as np
import cv2
import argparse

# Python 3 moved urlopen to urllib.requests
try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen

class NewOpenCVTests(unittest.TestCase):

    # path to local repository folder containing 'samples' folder
    repoPath = None
    # github repository url
    repoUrl = 'https://raw.github.com/Itseez/opencv/master'

    def get_sample(self, filename, iscolor = cv2.IMREAD_COLOR):
        if not filename in self.image_cache:
            filedata = None
            if NewOpenCVTests.repoPath is not None:
                candidate = NewOpenCVTests.repoPath + '/' + filename
                if os.path.isfile(candidate):
                    with open(candidate, 'rb') as f:
                        filedata = f.read()
            if filedata is None:
                filedata = urlopen(NewOpenCVTests.repoUrl + '/' + filename).read()
            self.image_cache[filename] = cv2.imdecode(np.fromstring(filedata, dtype=np.uint8), iscolor)
        return self.image_cache[filename]

    def setUp(self):
        self.image_cache = {}

    def hashimg(self, im):
        """ Compute a hash for an image, useful for image comparisons """
        return hashlib.md5(im.tostring()).digest()

    if sys.version_info[:2] == (2, 6):
        def assertLess(self, a, b, msg=None):
            if not a < b:
                self.fail('%s not less than %s' % (repr(a), repr(b)))

        def assertLessEqual(self, a, b, msg=None):
            if not a <= b:
                self.fail('%s not less than or equal to %s' % (repr(a), repr(b)))

        def assertGreater(self, a, b, msg=None):
            if not a > b:
                self.fail('%s not greater than %s' % (repr(a), repr(b)))

# Tests to run first; check the handful of basic operations that the later tests rely on

class Hackathon244Tests(NewOpenCVTests):

    def test_int_array(self):
        a = np.array([-1, 2, -3, 4, -5])
        absa0 = np.abs(a)
        self.assertTrue(cv2.norm(a, cv2.NORM_L1) == 15)
        absa1 = cv2.absdiff(a, 0)
        self.assertEqual(cv2.norm(absa1, absa0, cv2.NORM_INF), 0)

    def test_imencode(self):
        a = np.zeros((480, 640), dtype=np.uint8)
        flag, ajpg = cv2.imencode("img_q90.jpg", a, [cv2.IMWRITE_JPEG_QUALITY, 90])
        self.assertEqual(flag, True)
        self.assertEqual(ajpg.dtype, np.uint8)
        self.assertGreater(ajpg.shape[0], 1)
        self.assertEqual(ajpg.shape[1], 1)

    def test_projectPoints(self):
        objpt = np.float64([[1,2,3]])
        imgpt0, jac0 = cv2.projectPoints(objpt, np.zeros(3), np.zeros(3), np.eye(3), np.float64([]))
        imgpt1, jac1 = cv2.projectPoints(objpt, np.zeros(3), np.zeros(3), np.eye(3), None)
        self.assertEqual(imgpt0.shape, (objpt.shape[0], 1, 2))
        self.assertEqual(imgpt1.shape, imgpt0.shape)
        self.assertEqual(jac0.shape, jac1.shape)
        self.assertEqual(jac0.shape[0], 2*objpt.shape[0])

    def test_estimateAffine3D(self):
        pattern_size = (11, 8)
        pattern_points = np.zeros((np.prod(pattern_size), 3), np.float32)
        pattern_points[:,:2] = np.indices(pattern_size).T.reshape(-1, 2)
        pattern_points *= 10
        (retval, out, inliers) = cv2.estimateAffine3D(pattern_points, pattern_points)
        self.assertEqual(retval, 1)
        if cv2.norm(out[2,:]) < 1e-3:
            out[2,2]=1
        self.assertLess(cv2.norm(out, np.float64([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]])), 1e-3)
        self.assertEqual(cv2.countNonZero(inliers), pattern_size[0]*pattern_size[1])

    def test_fast(self):
        fd = cv2.FastFeatureDetector_create(30, True)
        img = self.get_sample("samples/data/right02.jpg", 0)
        img = cv2.medianBlur(img, 3)
        imgc = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
        keypoints = fd.detect(img)
        self.assertTrue(600 <= len(keypoints) <= 700)
        for kpt in keypoints:
            self.assertNotEqual(kpt.response, 0)

    def check_close_angles(self, a, b, angle_delta):
        self.assertTrue(abs(a - b) <= angle_delta or
                        abs(360 - abs(a - b)) <= angle_delta)

    def check_close_pairs(self, a, b, delta):
        self.assertLessEqual(abs(a[0] - b[0]), delta)
        self.assertLessEqual(abs(a[1] - b[1]), delta)

    def check_close_boxes(self, a, b, delta, angle_delta):
        self.check_close_pairs(a[0], b[0], delta)
        self.check_close_pairs(a[1], b[1], delta)
        self.check_close_angles(a[2], b[2], angle_delta)

    def test_geometry(self):
        npt = 100
        np.random.seed(244)
        a = np.random.randn(npt,2).astype('float32')*50 + 150

        img = np.zeros((300, 300, 3), dtype='uint8')
        be = cv2.fitEllipse(a)
        br = cv2.minAreaRect(a)
        mc, mr = cv2.minEnclosingCircle(a)

        be0 = ((150.2511749267578, 150.77322387695312), (158.024658203125, 197.57696533203125), 37.57804489135742)
        br0 = ((161.2974090576172, 154.41793823242188), (199.2301483154297, 207.7177734375), -9.164555549621582)
        mc0, mr0 = (160.41790771484375, 144.55152893066406), 136.713500977

        self.check_close_boxes(be, be0, 5, 15)
        self.check_close_boxes(br, br0, 5, 15)
        self.check_close_pairs(mc, mc0, 5)
        self.assertLessEqual(abs(mr - mr0), 5)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='run OpenCV python tests')
    parser.add_argument('--repo', help='use sample image files from local git repository (path to folder), '
                                       'if not set, samples will be downloaded from github.com')
    parser.add_argument('--data', help='<not used> use data files from local folder (path to folder), '
                                        'if not set, data files will be downloaded from docs.opencv.org')
    args, other = parser.parse_known_args()
    print("Testing OpenCV", cv2.__version__)
    print("Local repo path:", args.repo)
    NewOpenCVTests.repoPath = args.repo
    random.seed(0)
    unit_argv = [sys.argv[0]] + other;
    unittest.main(argv=unit_argv)
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys, getopt
from google_drive import *
import datetime

class File:
	def __init__(self, name, path, isFolder, timestamp, size):
		self.name = name
		self.path = path
		self.isFolder = isFolder
		self.timestamp = timestamp
		self.size = size

def syncFolder(baseFolder, localFolderPath, remoteFolder, exclude, allowIncrease = True):
	print('Checking Folder: ' + localFolderPath)

	''' get local files '''
	localFiles = {}
	for localFileName in os.listdir(localFolderPath):
		localFilePath = os.path.join(localFolderPath, localFileName)
		include = True
		for e in exclude:
			if localFilePath.startswith(baseFolder + '\\' + e):
				include = False
		if include:
			localFiles[localFileName] = File(name = localFileName, path = localFilePath, isFolder = not os.path.isfile(localFilePath), timestamp = int(os.path.getmtime(localFilePath)), size = os.stat(localFilePath).st_size)

	''' get remote files '''
	remoteFiles = remoteFolder.list()

	''' remove any duplicate folders or files '''
	remoteFilesToTrash = []
	for remoteFileName in sorted(remoteFiles.keys()):
		remoteFile = remoteFiles[remoteFileName]
		if len(remoteFile.ids) > 1:
			print('Trashing Duplicate Files: ' + localFolderPath + '\\' + remoteFile.name)
			remoteFile.trashSelf()
			remoteFilesToTrash.append(remoteFile.name)
	for remoteFileName in remoteFilesToTrash:
		del remoteFiles[remoteFileName]

	remoteFilesToTrash = []
	''' remove any excess remote folders or files '''
	for remoteFileName in sorted(remoteFiles.keys()):
		remoteFile = remoteFiles[remoteFileName]
		if remoteFile.isFolder:
			if remoteFile.name not in localFiles:
				print('Trashing Folder: ' + localFolderPath + '\\' + remoteFile.name)
				remoteFile.trashSelf()
				remoteFilesToTrash.append(remoteFile.name)
		else:
			if remoteFile.name not in localFiles:
				print('Trashing File: ' + localFolderPath + '\\' + remoteFile.name)
				remoteFile.trashSelf()
				remoteFilesToTrash.append(remoteFile.name)
	for remoteFileName in remoteFilesToTrash:
		del remoteFiles[remoteFileName]

	''' sync local files to remote files '''
	for localFileName in sorted(localFiles.keys()):
		localFile = localFiles[localFileName]
		if localFile.isFolder:
			if localFile.name not in remoteFiles:
				if allowIncrease:
					print('Creating Folder: ' + localFile.path)
					remoteFiles[localFile.name] = remoteFolder.createFolder(localFile.name)
		else:
			if localFile.name not in remoteFiles:
				if allowIncrease:
					print('Creating File: ' + localFile.path)
					remoteFolder.createFile(localFile.path, localFile.name, localFile.timestamp)
			else:
				if localFile.timestamp != remoteFiles[localFile.name].timestamp:
					if allowIncrease or localFile.size >= remoteFiles[localFile.name].size:
						oldTime = datetime.datetime.fromtimestamp(remoteFiles[localFile.name].timestamp).strftime('%Y-%m-%d %H:%M:%S')
						newTime = datetime.datetime.fromtimestamp(localFile.timestamp).strftime('%Y-%m-%d %H:%M:%S')
						print('Updating File: ' + localFile.path + '(' + oldTime + ' -> ' + newTime + ')')
						remoteFolder.updateFile(localFile.path, remoteFiles[localFile.name], localFile.timestamp)

	''' recurse into sub folders '''
	for localFileName in sorted(localFiles.keys()):
		localFile = localFiles[localFileName]
		if localFile.isFolder and localFile.name in remoteFiles:
			syncFolder(baseFolder, localFile.path, remoteFiles[localFile.name], exclude, allowIncrease)

if __name__ == '__main__':
	help = 'See README.md for information. Syntax: backup.py -l <local folder> -d <drive folder> [-e <exclude folder>]* [--no-increase]'
	baseFolder = ''
	localFolder = ''
	driveFolder = ''
	allowIncrease = True
	exclude = []
	try:
		opts, args = getopt.getopt(sys.argv[1:],'hl:d:e:',['help', 'local-folder=', 'drive-folder=', 'exclude=', 'no-increase'])
	except getopt.GetoptError:
		print(help)
		sys.exit(-1)
	for opt, arg in opts:
		if opt in ('-h', '--help'):
			print(help)
			sys.exit()
		elif opt in ("-l", "--local-folder"):
			localFolder = arg
			baseFolder = localFolder
		elif opt in ("-d", "--drive-folder"):
			driveFolder = arg
		elif opt in ("-e", "--exclude"):
			exclude.append(arg)
		elif opt == '--no-increase':
			allowIncrease = False
	if localFolder == '' or driveFolder == '':
		print(help)
		sys.exit(-1)
	googleDrive = GoogleDrive()
	syncFolder(baseFolder, localFolder, googleDrive.root().getAndCreateFolderPath(driveFolder), exclude, allowIncrease)
	sys.exit(0)<commit_msg>Made it more robust against problems<commit_after>#!/usr/bin/python

import sys, getopt
from google_drive import *
import datetime

class File:
	def __init__(self, name, path, isFolder, timestamp, size):
		self.name = name
		self.path = path
		self.isFolder = isFolder
		self.timestamp = timestamp
		self.size = size

def syncFolder(baseFolder, localFolderPath, remoteFolder, exclude, allowIncrease = True):
	print('Checking Folder: ' + localFolderPath)

	''' get local files '''
	localFiles = {}
	try:
		for localFileName in os.listdir(localFolderPath):
			localFilePath = os.path.join(localFolderPath, localFileName)
			include = True
			for e in exclude:
				if localFilePath.startswith(baseFolder + '\\' + e):
					include = False
			if include:
				try:
					localFiles[localFileName] = File(name = localFileName, path = localFilePath, isFolder = not os.path.isfile(localFilePath), timestamp = int(os.path.getmtime(localFilePath)), size = os.stat(localFilePath).st_size)
				except:
					print('Could not get info on file: ' + localFileName)
	except:
		print('Could not get info on directory: ' + localFolderPath)

	''' get remote files '''
	try:
		remoteFiles = remoteFolder.list()
	except:
		remoteFiles = {}

	''' remove any duplicate folders or files '''
	remoteFilesToTrash = []
	for remoteFileName in sorted(remoteFiles.keys()):
		remoteFile = remoteFiles[remoteFileName]
		if len(remoteFile.ids) > 1:
			print('Trashing Duplicate Files: ' + localFolderPath + '\\' + remoteFile.name)
			try:
				remoteFile.trashSelf()
				remoteFilesToTrash.append(remoteFile.name)
			except:
				print('Could not trash file: ' + localFolderPath + '\\' + remoteFile.name)
	for remoteFileName in remoteFilesToTrash:
		del remoteFiles[remoteFileName]

	remoteFilesToTrash = []
	''' remove any excess remote folders or files '''
	for remoteFileName in sorted(remoteFiles.keys()):
		remoteFile = remoteFiles[remoteFileName]
		if remoteFile.isFolder:
			if remoteFile.name not in localFiles:
				print('Trashing Folder: ' + localFolderPath + '\\' + remoteFile.name)
				try:
					remoteFile.trashSelf()
					remoteFilesToTrash.append(remoteFile.name)
				except:
					print('Could not trash folder: ' + localFolderPath + '\\' + remoteFile.name)
		else:
			if remoteFile.name not in localFiles:
				print('Trashing File: ' + localFolderPath + '\\' + remoteFile.name)
				try:
					remoteFile.trashSelf()
					remoteFilesToTrash.append(remoteFile.name)
				except:
					print('Could not trash file: ' + localFolderPath + '\\' + remoteFile.name)
	for remoteFileName in remoteFilesToTrash:
		del remoteFiles[remoteFileName]

	''' sync local files to remote files '''
	for localFileName in sorted(localFiles.keys()):
		localFile = localFiles[localFileName]
		if localFile.isFolder:
			if localFile.name not in remoteFiles:
				if allowIncrease:
					print('Creating Folder: ' + localFile.path)
					try:
						remoteFiles[localFile.name] = remoteFolder.createFolder(localFile.name)
					except:
						print('Could not create folder: ' + localFile.path)
		else:
			if localFile.name not in remoteFiles:
				if allowIncrease:
					print('Creating File: ' + localFile.path)
					try:
						remoteFolder.createFile(localFile.path, localFile.name, localFile.timestamp)
					except:
						print('Could not create file: ' + localFile.path)
			else:
				if localFile.timestamp != remoteFiles[localFile.name].timestamp:
					if allowIncrease or localFile.size >= remoteFiles[localFile.name].size:
						oldTime = datetime.datetime.fromtimestamp(remoteFiles[localFile.name].timestamp).strftime('%Y-%m-%d %H:%M:%S')
						newTime = datetime.datetime.fromtimestamp(localFile.timestamp).strftime('%Y-%m-%d %H:%M:%S')
						print('Updating File: ' + localFile.path + ' (' + oldTime + ' -> ' + newTime + ')')
						try:
							remoteFolder.updateFile(localFile.path, remoteFiles[localFile.name], localFile.timestamp)
						except:
							print('Could not update file: ' + localFile.path)

	''' recurse into sub folders '''
	for localFileName in sorted(localFiles.keys()):
		localFile = localFiles[localFileName]
		if localFile.isFolder and localFile.name in remoteFiles:
			syncFolder(baseFolder, localFile.path, remoteFiles[localFile.name], exclude, allowIncrease)

if __name__ == '__main__':
	help = 'See README.md for information. Syntax: backup.py -l <local folder> -d <drive folder> [-e <exclude folder>]* [--no-increase]'
	baseFolder = ''
	localFolder = ''
	driveFolder = ''
	allowIncrease = True
	exclude = []
	try:
		opts, args = getopt.getopt(sys.argv[1:],'hl:d:e:',['help', 'local-folder=', 'drive-folder=', 'exclude=', 'no-increase'])
	except getopt.GetoptError:
		print(help)
		sys.exit(-1)
	for opt, arg in opts:
		if opt in ('-h', '--help'):
			print(help)
			sys.exit()
		elif opt in ("-l", "--local-folder"):
			localFolder = arg
			baseFolder = localFolder
		elif opt in ("-d", "--drive-folder"):
			driveFolder = arg
		elif opt in ("-e", "--exclude"):
			exclude.append(arg)
		elif opt == '--no-increase':
			allowIncrease = False
	if localFolder == '' or driveFolder == '':
		print(help)
		sys.exit(-1)
	googleDrive = GoogleDrive()
	remoteFolder = googleDrive.root().getAndCreateFolderPath(driveFolder)
	syncFolder(baseFolder, localFolder, remoteFolder, exclude, allowIncrease)

	# write timestamp file
	localFilePath = localFolder + '\\last_backup.txt'
	localTime = datetime.datetime.now()
	with open(localFilePath, 'a') as f:
		f.write(str(localTime) + '\n')
	remoteFolder.createFile(localFilePath, 'last_backup.txt', int(os.path.getmtime(localFilePath)))
	input('Press ENTER to exit')<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'nwilson'

import threading
import os
import datetime

LOG_DATE_FORMAT = "%H:%M:%S"
LOGGER_ENV = "GIDGET_LOG_FILE"

def log(tag, msg):
    Logger._getOrCreate().log(tag, msg)

class Logger:
    _loggerThisProcess = None

    def __init__(self, logFilePath):
        self.fdLog = open(logFilePath, "a")

    def log(self, tag, msg):
        self.fdLog.write("[%s] %s %s\n" % (datetime.now().strftime(LOG_DATE_FORMAT), tag, msg.strip()))

    def close(self):
        self.fdLog.close()

    @staticmethod
    def _getOrCreate():
        if Logger._loggerThisProcess is not None:
            return Logger._loggerThisProcess

        assert os.getenv(LOGGER_ENV) is not None

        Logger._loggerThisProcess = Logger(os.getenv(LOGGER_ENV))
        return Logger._loggerThisProcess

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.logFile.closed():
            self.logFile.close()

class LogPipe(threading.Thread):
    def __init__(self, tag, logger):
        """
        Setup the object with a logger
        and start the thread
        """
        threading.Thread.__init__(self)
        self.daemon = False
        self.tag = tag
        self.fdRead, self.fdWrite = os.pipe()
        self.pipeReader = os.fdopen(self.fdRead)
        self.logger = logger
        self.start()

    def fileno(self):
        """
        Return the write file descriptor of the pipe
        """
        return self.fdWrite

    def run(self):
        """
        Run the thread, logging everything.
        """
        for line in iter(self.pipeReader.readline, ''):
            self.logger.log(self.tag, line)

        self.pipeReader.close()

    def close(self):
        """
        Close the write end of the pipe.
        """
        # TODO
        os.close(self.fdWrite)
        self.join()
        self.pipeReader.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
<commit_msg>Add ability to log from the command line<commit_after>#!/usr/bin/env python

__author__ = 'nwilson'

import threading
import os
import datetime
import sys

LOG_DATE_FORMAT = "%H:%M:%S"
LOGGER_ENV = "GIDGET_LOG_FILE"

def log(tag, msg):
    Logger._getOrCreate().log(tag, msg)

class Logger:
    _loggerThisProcess = None

    def __init__(self, logFilePath):
        self.fdLog = open(logFilePath, "a")

    def log(self, tag, msg):
        self.fdLog.write("[%s] %s %s\n" % (datetime.now().strftime(LOG_DATE_FORMAT), tag, msg.strip()))

    def close(self):
        self.fdLog.close()

    @staticmethod
    def _getOrCreate():
        if Logger._loggerThisProcess is not None:
            return Logger._loggerThisProcess

        assert os.getenv(LOGGER_ENV) is not None

        Logger._loggerThisProcess = Logger(os.getenv(LOGGER_ENV))
        return Logger._loggerThisProcess

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.logFile.closed():
            self.logFile.close()

class LogPipe(threading.Thread):
    def __init__(self, tag, logger):
        """
        Setup the object with a logger
        and start the thread
        """
        threading.Thread.__init__(self)
        self.daemon = False
        self.tag = tag
        self.fdRead, self.fdWrite = os.pipe()
        self.pipeReader = os.fdopen(self.fdRead)
        self.logger = logger
        self.start()

    def fileno(self):
        """
        Return the write file descriptor of the pipe
        """
        return self.fdWrite

    def run(self):
        """
        Run the thread, logging everything.
        """
        for line in iter(self.pipeReader.readline, ''):
            self.logger.log(self.tag, line)

        self.pipeReader.close()

    def close(self):
        """
        Close the write end of the pipe.
        """
        # TODO
        os.close(self.fdWrite)
        self.join()
        self.pipeReader.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

# for use when logging is needed from a shell script or Perl code
if __name__ == "__main__":

    if len(sys.argv) != 3:
        sys.stderr.write("Usage: python %s <tag> <message>" % sys.argv[0])
        exit(1)

    log(sys.argv[1], sys.argv[2])<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (c) 2014 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from anybox.testing.openerp import SharedSetupTransactionCase

class test_duplicate_analysis_report(SharedSetupTransactionCase):

    _data_files = (
        'data/users_demo.xml',
    )

    _module_ns = 'ecolo'

    def setUp(self):
        super(test_duplicate_analysis_report, self).setUp()

        self.env['ir.model'].clear_caches()
        self.env['ir.model.data'].clear_caches()
        self.partner_obj = self.env['res.partner']
        self.user_obj = self.env['res.users']
        self.email_coordinate_obj = self.env['email.coordinate']
        self.dar_obj = self.env['duplicate.analysis.report']
        self.group_obj = self.env['res.groups']

    def test_get_partner_ids(self):
        partner_ids = self.dar_obj._get_partner_ids()
        self.assertTrue(len(partner_ids) > 0, 'Should find a partner')
        partner = self.env.ref('%s.partner_of_user_test' % self._module_ns)
        self.assertTrue(
            partner.id in partner_ids,
            'Configurator should be into the recipients')
<commit_msg>[mozaik_person_coordinate] refs #14353 [FIX] header<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#     This file is part of mozaik_person_coordinate, an Odoo module.
#
#     Copyright (c) 2015 ACSONE SA/NV (<http://acsone.eu>)
#
#     mozaik_person_coordinate is free software:
#     you can redistribute it and/or
#     modify it under the terms of the GNU Affero General Public License
#     as published by the Free Software Foundation, either version 3 of
#     the License, or (at your option) any later version.
#
#     mozaik_person_coordinate is distributed in the hope that it will
#     be useful but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the
#     GNU Affero General Public License
#     along with mozaik_person_coordinate.
#     If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from anybox.testing.openerp import SharedSetupTransactionCase

class test_duplicate_analysis_report(SharedSetupTransactionCase):

    _data_files = (
        'data/users_demo.xml',
    )

    _module_ns = 'ecolo'

    def setUp(self):
        super(test_duplicate_analysis_report, self).setUp()

        self.env['ir.model'].clear_caches()
        self.env['ir.model.data'].clear_caches()
        self.partner_obj = self.env['res.partner']
        self.user_obj = self.env['res.users']
        self.email_coordinate_obj = self.env['email.coordinate']
        self.dar_obj = self.env['duplicate.analysis.report']
        self.group_obj = self.env['res.groups']

    def test_get_partner_ids(self):
        partner_ids = self.dar_obj._get_partner_ids()
        self.assertTrue(len(partner_ids) > 0, 'Should find a partner')
        partner = self.env.ref('%s.partner_of_user_test' % self._module_ns)
        self.assertTrue(
            partner.id in partner_ids,
            'Configurator should be into the recipients')
<|endoftext|>