<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download, unzip and prep the latest CAL-ACCESS database ZIP.
"""
from __future__ import unicode_literals
import os
import shutil
import zipfile
import requests
import calaccess_raw
from datetime import datetime
from hurry.filesize import size
from clint.textui import progress
from django.conf import settings
from django.core.files import File
from django.utils.timezone import utc, now
from calaccess_raw import get_download_directory
from django.template.loader import render_to_string
from django.core.management.base import CommandError
from calaccess_raw.management.commands import CalAccessCommand
from django.contrib.humanize.templatetags.humanize import naturaltime
from calaccess_raw.models.tracking import RawDataVersion, RawDataFile

class Command(CalAccessCommand):
    """
    Download, unzip and prep the latest CAL-ACCESS database ZIP.
    """
    help = "Download, unzip and prep the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--keep-files",
            action="store_true",
            dest="keep_files",
            default=False,
            help="Keep downloaded zip and unzipped files"
        )
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Download the ZIP archive without asking permission"
        )
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # get the dir where data goes from app settings
        self.data_dir = get_download_directory()
        # if data_dir doesn't exist, create it
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)

        # downloaded zip file will go in data_dir
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])
        # raw tsv files go in same data_dir in tsv/
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")

        download_metadata = self.get_download_metadata()

        self.current_release_datetime = download_metadata['last-modified']
        self.current_release_size = download_metadata['content-length']

        self.last_started_download = self.get_last_log()
        self.last_finished_download = self.get_last_log(finished=True)

        if self.last_finished_download:
            last_release_datetime = self.last_finished_download.version.release_datetime
            since_prev_version = naturaltime(last_release_datetime)
        else:
            last_release_datetime = None
            since_prev_version = None

        if last_release_datetime == self.current_release_datetime:
            already_downloaded = True
        else:
            already_downloaded = False

        # can resume only if possible and not forcing re-start
        self.resume_download = self.check_can_resume() and not options['restart']

        if self.resume_download:
            # set current size to partially downloaded zip
            self.local_file_size = os.path.getsize(self.zip_path)
            # set the datetime of last download to last modified date
            # of zip file
            timestamp = os.path.getmtime(self.zip_path)
            self.local_file_datetime = datetime.fromtimestamp(timestamp, utc)
        else:
            self.local_file_size = 0
            self.local_file_datetime = None

        if not options['noinput'] and not options['restart']:

            # setting up the prompt
            prompt_context = dict(
                current_release_datetime=self.current_release_datetime,
                resuming=self.resume_download,
                already_downloaded=already_downloaded,
                expected_size=size(self.current_release_size),
                local_file_size=size(self.local_file_size),
                download_dir=self.data_dir,
                since_prev_version=since_prev_version,
                since_local_file_modified=naturaltime(self.local_file_datetime)
            )

            prompt = render_to_string(
                'calaccess_raw/downloadcalaccessrawdata.txt',
                prompt_context,
            )

            if not self.confirm_proceed(prompt):
                raise CommandError("Download cancelled")

        if self.resume_download:
            self.log_record = self.last_started_download
            self.version = self.log_record.version
        else:
            # get or create a version record
            # .get_or_create() throws IntegrityError
            try:
                self.version = self.raw_data_versions.get(
                    release_datetime=self.current_release_datetime
                )
            except RawDataVersion.DoesNotExist:
                self.version = self.raw_data_versions.create(
                    release_datetime=self.current_release_datetime,
                    size=download_metadata['content-length']
                )
            # create a log record
            self.log_record = self.command_logs.create(
                version=self.version,
                command=self,
                called_by=self.get_caller_log()
            )

        self.download()
        self.unzip()
        self.prep()
        self.track_files()

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()

        if not options['keep_files']:
            os.remove(self.zip_path)
            shutil.rmtree(os.path.join(self.data_dir, 'CalAccess'))

        self.log_record.finish_datetime = now()
        self.log_record.save()

    def check_can_resume(self):
        """
        Run a series of checks to see if the previous download can be resumed.

        If so, return True, else False.
        """
        result = False
        # if there's a zip file
        if os.path.exists(self.zip_path):
            # and there's a previous download
            if self.last_started_download:
                # that did not finish
                if not self.last_started_download.finish_datetime:

                    prev_release = self.last_started_download.version.release_datetime

                    # and the current release datetime is the same as
                    #  the one on the last incomplete download
                    if self.current_release_datetime == prev_release:
                        result = True
        return result

    def download(self):
        """
        Download the ZIP file in pieces.
        """
        if self.verbosity:
            if self.resume_download:
                self.header("Resuming download of ZIP file")
            else:
                self.header("Downloading ZIP file")

        expected_size = self.current_release_size

        # Prep
        headers = dict()
        if os.path.exists(self.zip_path):
            if self.resume_download:
                headers['Range'] = 'bytes=%d-' % self.local_file_size
                expected_size = expected_size - self.local_file_size
            else:
                os.remove(self.zip_path)

        # Stream the download
        chunk_size = 1024
        req = requests.get(self.url, stream=True, headers=headers)
        n_iters = float(expected_size) / chunk_size + 1
        with open(self.zip_path, 'ab') as fp:
            for chunk in progress.bar(req.iter_content(chunk_size=chunk_size),
                                      expected_size=n_iters):
                fp.write(chunk)
                fp.flush()

    def unzip(self):
        """
        Unzip the snapshot file.
        """
        if self.verbosity:
            self.log(" Unzipping archive")

        with zipfile.ZipFile(self.zip_path) as zf:
            for member in zf.infolist():
                words = member.filename.split('/')
                path = self.data_dir
                for word in words[:-1]:
                    drive, word = os.path.splitdrive(word)
                    head, word = os.path.split(word)
                    if word in (os.curdir, os.pardir, ''):
                        continue
                    path = os.path.join(path, word)
                zf.extract(member, path)

    def prep(self):
        """
        Rearrange the unzipped files and get rid of the stuff we don't want.
        """
        if self.verbosity:
            self.log(" Prepping unzipped data")

        # Move the deep down directory we want out
        shutil.move(
            os.path.join(
                self.data_dir,
                'CalAccess/DATA/CalAccess/DATA/'
            ),
            self.data_dir
        )
        # Clear out target if it exists
        if os.path.exists(self.tsv_dir):
            shutil.rmtree(self.tsv_dir)

        # Rename it to the target
        shutil.move(
            os.path.join(self.data_dir, "DATA/"),
            self.tsv_dir,
        )

    def track_files(self):
        """
        Create a RawDataFile for each download .TSV file.
        """
        for f in os.listdir(self.tsv_dir):
            if '.TSV' in f:
                file_name = f.upper().replace('.TSV', '')
                self.raw_data_files.get_or_create(
                    version=self.version,
                    file_name=file_name,
                )

    def archive(self):
        """
        Save a copy of the download zip file and each file inside.
        """
        if self.verbosity:
            self.log(" Archiving original files")
        # Remove previous zip file
        self.version.zip_file_archive.delete()
        # Open up the zipped file so we can wrap it in the Django File obj
        zipped_file = open(self.zip_path)
        # Save the zip on the raw data version
        self.version.zip_file_archive.save(
            self.url.split('/')[-1],
            File(zipped_file)
        )
        zipped_file.close()

        # make the RawDataFile records
        for raw_data_file in self.raw_data_files.filter(version=self.version):
            if self.verbosity > 2:
                self.log(" Archiving {0}.TSV".format(raw_data_file.file_name))
            # Remove previous .TSV file
            raw_data_file.download_file_archive.delete()
            # Open up the .TSV file so we can wrap it in the Django File obj
            with open(self.tsv_dir + raw_data_file.file_name + '.TSV') as f:
                # Save the .TSV on the raw data file
                raw_data_file.download_file_archive.save(
                    raw_data_file.file_name + '.TSV',
                    File(f)
                )

class TestCommand(Command):
    """
    Simulates downloading and unzipping of CAL-ACCESS raw data for testing.
    """
    help = "Simulates downloading and unzipping of CAL-ACCESS raw data for testing"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(TestCommand, self).add_arguments(parser)

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        self.verbosity = options.get("verbosity")
        self.no_color = options.get("no_color")
        self.raw_data_files = RawDataFile.objects
        self.data_dir = calaccess_raw.get_test_download_directory()
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])

        with open(self.data_dir + "/sampled_version.txt", "r") as f:
            release_datetime = f.readline()
            size = f.readline()

        try:
            self.version = RawDataVersion.objects.get(
                release_datetime=release_datetime
            )
        except RawDataVersion.DoesNotExist:
            self.version = RawDataVersion.objects.create(
                release_datetime=release_datetime,
                size=size
            )

        self.unzip()
        self.prep()
        self.track_files()

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()
<commit_msg>At high verbosity, log when the zip file is being archived<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download, unzip and prep the latest CAL-ACCESS database ZIP.
"""
from __future__ import unicode_literals
import os
import shutil
import zipfile
import requests
import calaccess_raw
from datetime import datetime
from hurry.filesize import size
from clint.textui import progress
from django.conf import settings
from django.core.files import File
from django.utils.timezone import utc, now
from calaccess_raw import get_download_directory
from django.template.loader import render_to_string
from django.core.management.base import CommandError
from calaccess_raw.management.commands import CalAccessCommand
from django.contrib.humanize.templatetags.humanize import naturaltime
from calaccess_raw.models.tracking import RawDataVersion, RawDataFile

class Command(CalAccessCommand):
    """
    Download, unzip and prep the latest CAL-ACCESS database ZIP.
    """
    help = "Download, unzip and prep the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--keep-files",
            action="store_true",
            dest="keep_files",
            default=False,
            help="Keep downloaded zip and unzipped files"
        )
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Download the ZIP archive without asking permission"
        )
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # get the dir where data goes from app settings
        self.data_dir = get_download_directory()
        # if data_dir doesn't exist, create it
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)

        # downloaded zip file will go in data_dir
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])
        # raw tsv files go in same data_dir in tsv/
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")

        download_metadata = self.get_download_metadata()

        self.current_release_datetime = download_metadata['last-modified']
        self.current_release_size = download_metadata['content-length']

        self.last_started_download = self.get_last_log()
        self.last_finished_download = self.get_last_log(finished=True)

        if self.last_finished_download:
            last_release_datetime = self.last_finished_download.version.release_datetime
            since_prev_version = naturaltime(last_release_datetime)
        else:
            last_release_datetime = None
            since_prev_version = None

        if last_release_datetime == self.current_release_datetime:
            already_downloaded = True
        else:
            already_downloaded = False

        # can resume only if possible and not forcing re-start
        self.resume_download = self.check_can_resume() and not options['restart']

        if self.resume_download:
            # set current size to partially downloaded zip
            self.local_file_size = os.path.getsize(self.zip_path)
            # set the datetime of last download to last modified date
            # of zip file
            timestamp = os.path.getmtime(self.zip_path)
            self.local_file_datetime = datetime.fromtimestamp(timestamp, utc)
        else:
            self.local_file_size = 0
            self.local_file_datetime = None

        if not options['noinput'] and not options['restart']:

            # setting up the prompt
            prompt_context = dict(
                current_release_datetime=self.current_release_datetime,
                resuming=self.resume_download,
                already_downloaded=already_downloaded,
                expected_size=size(self.current_release_size),
                local_file_size=size(self.local_file_size),
                download_dir=self.data_dir,
                since_prev_version=since_prev_version,
                since_local_file_modified=naturaltime(self.local_file_datetime)
            )

            prompt = render_to_string(
                'calaccess_raw/downloadcalaccessrawdata.txt',
                prompt_context,
            )

            if not self.confirm_proceed(prompt):
                raise CommandError("Download cancelled")

        if self.resume_download:
            self.log_record = self.last_started_download
            self.version = self.log_record.version
        else:
            # get or create a version record
            # .get_or_create() throws IntegrityError
            try:
                self.version = self.raw_data_versions.get(
                    release_datetime=self.current_release_datetime
                )
            except RawDataVersion.DoesNotExist:
                self.version = self.raw_data_versions.create(
                    release_datetime=self.current_release_datetime,
                    size=download_metadata['content-length']
                )
            # create a log record
            self.log_record = self.command_logs.create(
                version=self.version,
                command=self,
                called_by=self.get_caller_log()
            )

        self.download()
        self.unzip()
        self.prep()
        self.track_files()

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()

        if not options['keep_files']:
            os.remove(self.zip_path)
            shutil.rmtree(os.path.join(self.data_dir, 'CalAccess'))

        self.log_record.finish_datetime = now()
        self.log_record.save()

    def check_can_resume(self):
        """
        Run a series of checks to see if the previous download can be resumed.

        If so, return True, else False.
        """
        result = False
        # if there's a zip file
        if os.path.exists(self.zip_path):
            # and there's a previous download
            if self.last_started_download:
                # that did not finish
                if not self.last_started_download.finish_datetime:

                    prev_release = self.last_started_download.version.release_datetime

                    # and the current release datetime is the same as
                    #  the one on the last incomplete download
                    if self.current_release_datetime == prev_release:
                        result = True
        return result

    def download(self):
        """
        Download the ZIP file in pieces.
        """
        if self.verbosity:
            if self.resume_download:
                self.header("Resuming download of ZIP file")
            else:
                self.header("Downloading ZIP file")

        expected_size = self.current_release_size

        # Prep
        headers = dict()
        if os.path.exists(self.zip_path):
            if self.resume_download:
                headers['Range'] = 'bytes=%d-' % self.local_file_size
                expected_size = expected_size - self.local_file_size
            else:
                os.remove(self.zip_path)

        # Stream the download
        chunk_size = 1024
        req = requests.get(self.url, stream=True, headers=headers)
        n_iters = float(expected_size) / chunk_size + 1
        with open(self.zip_path, 'ab') as fp:
            for chunk in progress.bar(req.iter_content(chunk_size=chunk_size),
                                      expected_size=n_iters):
                fp.write(chunk)
                fp.flush()

    def unzip(self):
        """
        Unzip the snapshot file.
        """
        if self.verbosity:
            self.log(" Unzipping archive")

        with zipfile.ZipFile(self.zip_path) as zf:
            for member in zf.infolist():
                words = member.filename.split('/')
                path = self.data_dir
                for word in words[:-1]:
                    drive, word = os.path.splitdrive(word)
                    head, word = os.path.split(word)
                    if word in (os.curdir, os.pardir, ''):
                        continue
                    path = os.path.join(path, word)
                zf.extract(member, path)

    def prep(self):
        """
        Rearrange the unzipped files and get rid of the stuff we don't want.
        """
        if self.verbosity:
            self.log(" Prepping unzipped data")

        # Move the deep down directory we want out
        shutil.move(
            os.path.join(
                self.data_dir,
                'CalAccess/DATA/CalAccess/DATA/'
            ),
            self.data_dir
        )
        # Clear out target if it exists
        if os.path.exists(self.tsv_dir):
            shutil.rmtree(self.tsv_dir)

        # Rename it to the target
        shutil.move(
            os.path.join(self.data_dir, "DATA/"),
            self.tsv_dir,
        )

    def track_files(self):
        """
        Create a RawDataFile for each download .TSV file.
        """
        for f in os.listdir(self.tsv_dir):
            if '.TSV' in f:
                file_name = f.upper().replace('.TSV', '')
                self.raw_data_files.get_or_create(
                    version=self.version,
                    file_name=file_name,
                )

    def archive(self):
        """
        Save a copy of the download zip file and each file inside.
        """
        if self.verbosity:
            self.log(" Archiving original files")
        if self.verbosity > 2:
            self.log(" Archiving {0}".format(os.path.basename(self.zip_path)))
        # Remove previous zip file
        self.version.zip_file_archive.delete()
        # Open up the zipped file so we can wrap it in the Django File obj
        zipped_file = open(self.zip_path)
        # Save the zip on the raw data version
        self.version.zip_file_archive.save(
            self.url.split('/')[-1],
            File(zipped_file)
        )
        zipped_file.close()

        # make the RawDataFile records
        for raw_data_file in self.raw_data_files.filter(version=self.version):
            if self.verbosity > 2:
                self.log(" Archiving {0}.TSV".format(raw_data_file.file_name))
            # Remove previous .TSV file
            raw_data_file.download_file_archive.delete()
            # Open up the .TSV file so we can wrap it in the Django File obj
            with open(self.tsv_dir + raw_data_file.file_name + '.TSV') as f:
                # Save the .TSV on the raw data file
                raw_data_file.download_file_archive.save(
                    raw_data_file.file_name + '.TSV',
                    File(f)
                )

class TestCommand(Command):
    """
    Simulates downloading and unzipping of CAL-ACCESS raw data for testing.
    """
    help = "Simulates downloading and unzipping of CAL-ACCESS raw data for testing"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(TestCommand, self).add_arguments(parser)

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        self.verbosity = options.get("verbosity")
        self.no_color = options.get("no_color")
        self.raw_data_files = RawDataFile.objects
        self.data_dir = calaccess_raw.get_test_download_directory()
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])

        with open(self.data_dir + "/sampled_version.txt", "r") as f:
            release_datetime = f.readline()
            size = f.readline()

        try:
            self.version = RawDataVersion.objects.get(
                release_datetime=release_datetime
            )
        except RawDataVersion.DoesNotExist:
            self.version = RawDataVersion.objects.create(
                release_datetime=release_datetime,
                size=size
            )

        self.unzip()
        self.prep()
        self.track_files()

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# encoding: utf-8

import os, sys
import Options, Task, Utils, Logs
from TaskGen import before, after, feature
from Constants import *

def detect(conf):
    if conf.check_cfg(path = 'gtest-config',
                      args = '--cppflags --cxxflags --ldflags --libs',
                      package = '',
                      uselib_store = 'gtest'):
        def f(str):
            if str == 'gtest':
                return 'gtest_main'
            else:
                return str
        conf.env.LIB_gtest = map(f, conf.env.LIB_gtest)

def set_options(opt):
    opt.add_option('--check', action = 'store_true', default = False,
                   help = 'Execute unit tests')
    opt.add_option('--checkall', action = 'store_true', default = False,
                   help = 'Execute all unit tests')

@feature('testt')
@feature('gtest')
@before('apply_core')
def test_remover(self):
    if not Options.options.check and not Options.options.checkall:
        self.meths[:] = []

@feature('testt')
@feature('gtest')
@after('apply_link', 'vars_target_cprogram')
def make_test(self):
    if not 'cprogram' in self.features:
        Logs.error('test cannot be executed %s'%self)
        return
    self.default_install_path = None
    self.create_task('utest', self.link_task.outputs)

@feature('gtest')
@before('apply_core')
def gtest_attach(self):
    if not self.env.HAVE_GTEST:
        Logs.error('gtest is not found')
        self.meths[:] = []
        return

    if isinstance(self.uselib, str):
        self.uselib += " gtest"
    else:
        self.uselib.append('gtest')

import threading
testlock = threading.Lock()

def exec_test(self):
    variant = self.env.variant()
    filename = self.inputs[0].abspath(self.env)

    try:
        fu = getattr(self.generator.bld, 'all_test_paths')
    except AttributeError:
        fu = os.environ.copy()
        self.generator.bld.all_test_paths = fu

        lst = []
        for obj in self.generator.bld.all_task_gen:
            link_task=getattr(obj, 'link_task', None)
            if link_task and link_task.env.variant()==variant:
                lst.append(link_task.outputs[0].parent.abspath(obj.env))

        def add_path(dct, path, var):
            dct[var] = os.pathsep.join(Utils.to_list(path)+[os.environ.get(var, '')])
        if sys.platform=='win32':
            add_path(fu, lst, 'PATH')
        elif sys.platform=='darwin':
            add_path(fu, lst, 'DYLD_LIBRARY_PATH')
            add_path(fu, lst, 'LD_LIBRARY_PATH')
        else:
            add_path(fu, lst, 'LD_LIBRARY_PATH')

    cwd = self.inputs[0].parent.abspath(self.env)
    proc = Utils.pproc.Popen(filename, cwd = cwd, env = fu,
                             stderr = Utils.pproc.PIPE,
                             stdout = Utils.pproc.PIPE)
    (stdout, stderr) = proc.communicate()
    tup = (filename, proc.returncode, stdout, stderr)

    testlock.acquire()
    try:
        Logs.debug("ut: %r",tup)
        try:
            self.generator.bld.utest_results.append(tup)
        except:
            self.generator.bld.utest_results = [tup]

        a = getattr(self.generator.bld, 'added_post_fun', False)
        if not a:
            self.generator.bld.add_post_fun(summary)
            self.generator.bld.added_post_fun = True

    finally:
        testlock.release()

cls = Task.task_type_from_func('utest', func = exec_test, color = 'PINK', ext_in = '.bin')
old = cls.runnable_status

def test_status(self):
    if Options.options.checkall:
        return RUN_ME
    return old(self)

cls.runnable_status = test_status
cls.quiet = 1

def summary(bld):
    lst = getattr(bld, 'utest_results', [])

    if not lst: return

    total = len(lst)
    fail = len([x for x in lst if x[1]])

    Utils.pprint('CYAN', 'test summary')
    Utils.pprint('CYAN', '  tests that pass %d/%d' % (total-fail, total))

    for (f, code, out, err) in lst:
        if not code:
            Utils.pprint('GREEN', '    %s' % f)

    if fail>0:
        Utils.pprint('RED', '  tests that pass %d/%d' % (fail, total))
        for (f, code, out, err) in lst:
            if code:
                Utils.pprint('RED', '    %s' % f)
                print(out)
<commit_msg>add option 'checkone'<commit_after>#! /usr/bin/env python
# encoding: utf-8

import os, sys
import Options, Task, Utils, Logs
from TaskGen import before, after, feature
from Constants import *

def detect(conf):
    if conf.check_cfg(path = 'gtest-config',
                      args = '--cppflags --cxxflags --ldflags --libs',
                      package = '',
                      uselib_store = 'gtest'):
        def f(str):
            if str == 'gtest':
                return 'gtest_main'
            else:
                return str
        conf.env.LIB_gtest = map(f, conf.env.LIB_gtest)

def set_options(opt):
    opt.add_option('--check', action = 'store_true', default = False,
                   help = 'Execute unit tests')
    opt.add_option('--checkall', action = 'store_true', default = False,
                   help = 'Execute all unit tests')
    opt.add_option('--checkone', action = 'store', default = False,
                   help = 'Execute specified unit test')

@feature('testt')
@feature('gtest')
@before('apply_core')
def test_remover(self):
    if not Options.options.check and not Options.options.checkall and self.target != Options.options.checkone:
        self.meths[:] = []

@feature('testt')
@feature('gtest')
@after('apply_link', 'vars_target_cprogram')
def make_test(self):
    if not 'cprogram' in self.features:
        Logs.error('test cannot be executed %s'%self)
        return
    self.default_install_path = None
    self.create_task('utest', self.link_task.outputs)

@feature('gtest')
@before('apply_core')
def gtest_attach(self):
    if not self.env.HAVE_GTEST:
        Logs.error('gtest is not found')
        self.meths[:] = []
        return

    if isinstance(self.uselib, str):
        self.uselib += " gtest"
    else:
        self.uselib.append('gtest')

import threading
testlock = threading.Lock()

def exec_test(self):
    variant = self.env.variant()
    filename = self.inputs[0].abspath(self.env)

    try:
        fu = getattr(self.generator.bld, 'all_test_paths')
    except AttributeError:
        fu = os.environ.copy()
        self.generator.bld.all_test_paths = fu

        lst = []
        for obj in self.generator.bld.all_task_gen:
            link_task=getattr(obj, 'link_task', None)
            if link_task and link_task.env.variant()==variant:
                lst.append(link_task.outputs[0].parent.abspath(obj.env))

        def add_path(dct, path, var):
            dct[var] = os.pathsep.join(Utils.to_list(path)+[os.environ.get(var, '')])
        if sys.platform=='win32':
            add_path(fu, lst, 'PATH')
        elif sys.platform=='darwin':
            add_path(fu, lst, 'DYLD_LIBRARY_PATH')
            add_path(fu, lst, 'LD_LIBRARY_PATH')
        else:
            add_path(fu, lst, 'LD_LIBRARY_PATH')

    cwd = self.inputs[0].parent.abspath(self.env)
    proc = Utils.pproc.Popen(filename, cwd = cwd, env = fu,
                             stderr = Utils.pproc.PIPE,
                             stdout = Utils.pproc.PIPE)
    (stdout, stderr) = proc.communicate()
    tup = (filename, proc.returncode, stdout, stderr)

    testlock.acquire()
    try:
        Logs.debug("ut: %r",tup)
        try:
            self.generator.bld.utest_results.append(tup)
        except:
            self.generator.bld.utest_results = [tup]

        a = getattr(self.generator.bld, 'added_post_fun', False)
        if not a:
            self.generator.bld.add_post_fun(summary)
            self.generator.bld.added_post_fun = True

    finally:
        testlock.release()

cls = Task.task_type_from_func('utest', func = exec_test, color = 'PINK', ext_in = '.bin')
old = cls.runnable_status

def test_status(self):
    if Options.options.checkall:
        return RUN_ME
    if Options.options.checkone == self.name:
        return RUN_ME
    return old(self)

cls.runnable_status = test_status
cls.quiet = 1

def summary(bld):
    lst = getattr(bld, 'utest_results', [])

    if not lst: return

    total = len(lst)
    fail = len([x for x in lst if x[1]])

    Utils.pprint('CYAN', 'test summary')
    Utils.pprint('CYAN', '  tests that pass %d/%d' % (total-fail, total))

    for (f, code, out, err) in lst:
        if not code:
            Utils.pprint('GREEN', '    %s' % f)

    if fail>0:
        Utils.pprint('RED', '  tests that fail %d/%d' % (fail, total))
        for (f, code, out, err) in lst:
            if code:
                Utils.pprint('RED', '    %s' % f)
                print(out)
<|endoftext|>


<commit_before>from __future__ import print_function
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
from torch.autograd import Variable
from data import VOC_CLASSES as labelmap
import torch.utils.data as data
from PIL import Image
import sys
import os
import argparse
from data import base_transform
import numpy as np
from ssd import build_ssd
import cv2

parser = argparse.ArgumentParser(description='Single Shot MultiBox Detection')
parser.add_argument('--trained_model', default='weights/', type=str, help='Trained state_dict file path to open')
parser.add_argument('--confidence_threshold', default=0.6, type=float, help='Detection confidence threshold')
parser.add_argument('--top_k', default=5, type=int, help='Further restrict the number of predictions to parse')
parser.add_argument('--cuda', default=False, type=bool, help='Use cuda to train model')
args = parser.parse_args()

args.trained_model = 'weights/ssd_300_VOC0712.pth'
net = build_ssd('test', 300, 21)    # initialize SSD
net.load_state_dict(torch.load(args.trained_model))
net.eval()
transform = base_transform(net.size,(104,117,123))
colors = [(255, 0, 0),(0, 255, 0),(0, 0, 255)]
font = cv2.FONT_HERSHEY_SIMPLEX

def predict(frame):
    #res = cv2.resize(img,(0.5*width, 0.5*height), interpolation = cv2.INTER_CUBIC)
    height,width = frame.shape[:2]
    img = Image.fromarray(frame)
    # print(img)
    x = Variable(transform(img).unsqueeze_(0))
    y = net(x)      # forward pass
    detections = y.data
    # scale each detection back up to the image
    scale = torch.Tensor([width,height,width,height])
    j = 0
    for i in range(detections.size(1)):
        while detections[0,i,j,0] >= 0.3:
            pt = (detections[0,i,j,1:]*scale).cpu().numpy()
            cv2.rectangle(frame, (int(pt[0]), int(pt[1])), (int(pt[2]), int(pt[3])), colors[i%3], 2)
            cv2.putText(frame, labelmap[i-1], (int(pt[0]), int(pt[1])), font, 2,(255,255,255),2,cv2.LINE_AA)
            # labels.append(label_name)
            j+=1
    return frame

video_capture = cv2.VideoCapture(0)

while True:
    if not video_capture.isOpened():
        print('Unable to load camera.')
        sleep(5)
        pass
    # Capture frame-by-frame
    ret, frame = video_capture.read()
    frame = predict(frame)
    # pts, labels = predict(frame)
    # color = (0, 255, 0)
    # font = cv2.FONT_HERSHEY_SIMPLEX
    # for i in range(len(pts)):
    #     cv2.rectangle(frame, (int(pts[i][0]), int(pts[i][1])), (int(pts[i][2]-pts[i][0]+1), int(pts[i][3]-pts[i][1]+1)), color, 2)
    #     cv2.putText(frame, labels[i], (pts[i][0], pts[i][1]), font, 4,(255,255,255),2,cv2.LINE_AA)
    # # Display the resulting frame
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    # Display the resulting frame
    cv2.imshow('Video', frame)
    cv2.waitKey(10)
# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()
<commit_msg>faster cpu webcam demo<commit_after>from __future__ import print_function
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
from torch.autograd import Variable
from data import VOC_CLASSES as labelmap
import torch.utils.data as data
from PIL import Image
import sys
import os
import argparse
from data import base_transform
import numpy as np
from ssd import build_ssd
import cv2

parser = argparse.ArgumentParser(description='Single Shot MultiBox Detection')
parser.add_argument('--trained_model', default='weights/', type=str, help='Trained state_dict file path to open')
parser.add_argument('--confidence_threshold', default=0.6, type=float, help='Detection confidence threshold')
parser.add_argument('--top_k', default=5, type=int, help='Further restrict the number of predictions to parse')
parser.add_argument('--cuda', default=False, type=bool, help='Use cuda to demo the model')
args = parser.parse_args()

args.trained_model = 'weights/ssd_300_VOC0712.pth'
net = build_ssd('test', 300, 21)    # initialize SSD
net.load_state_dict(torch.load(args.trained_model))
net.eval()
transform = base_transform(net.size,(104,117,123))
colors = [(255, 0, 0),(0, 255, 0),(0, 0, 255)]
font = cv2.FONT_HERSHEY_SIMPLEX

def predict(frame):
    height,width = frame.shape[:2]
    img = Image.fromarray(frame)
    x = Variable(transform(img).unsqueeze_(0))
    y = net(x)      # forward pass
    detections = y.data
    # scale each detection back up to the image
    scale = torch.Tensor([width,height,width,height])

    for i in range(detections.size(1)):
        j = 0
        while detections[0,i,j,0] >= 0.6:
            pt = (detections[0,i,j,1:]*scale).cpu().numpy()
            cv2.rectangle(frame, (int(pt[0]), int(pt[1])), (int(pt[2]), int(pt[3])), colors[i%3], 2)
            cv2.putText(frame, labelmap[i-1], (int(pt[0]), int(pt[1])), font, 2,(255,255,255),2,cv2.LINE_AA)
            # labels.append(label_name)
            j+=1
    return frame

video_capture = cv2.VideoCapture(0)
while True:
    if not video_capture.isOpened():
        print('Unable to load camera.')
        sleep(5)
        pass
    # Capture frame-by-frame
    ret, frame = video_capture.read()
    frame = predict(frame)
#    if cv2.waitKey(1) & 0xFF == ord('q'):
#        break
    # Display the resulting frame
    cv2.imshow('Video', frame)
    cv2.waitKey(5)
# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Copyright (C) 2012-2015  Diego Torres Milano
Created on Feb 2, 2015

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@author: Diego Torres Milano
'''

__version__ = '12.1.2'

import os
import subprocess
import sys
import platform
import threading
import re
import json
try:
    import requests
    REQUESTS_AVAILABLE = True
except:
    REQUESTS_AVAILABLE = False
import time
from com.dtmilano.android.adb.adbclient import AdbClient
from com.dtmilano.android.common import obtainAdbPath

__author__ = 'diego'

DEBUG = False

lock = threading.Lock()

class RunTestsThread(threading.Thread):
    """
    Runs the instrumentation for the specified package in a new thread.
    """
    def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None, adbClient=None, testClass=None, testRunner=None):
        threading.Thread.__init__(self, group=group, target=target, name=name, verbose=verbose)
        self.adbClient = adbClient
        self.testClass = testClass
        self.testRunner = testRunner
        self.pkg = re.sub('\.test$', '', self.testClass)

    def run(self):
        if DEBUG:
            print >> sys.stderr, "RunTestsThread: Acquiring lock"
        lock.acquire()
        if DEBUG:
            print >> sys.stderr, "RunTestsThread: Lock acquired"
        self.forceStop()
        time.sleep(3)
        if DEBUG:
            print >> sys.stderr, "Starting test..."
            print >> sys.stderr, "RunTestsThread: Releasing lock"
        lock.release()
        out = self.adbClient.shell('am instrument -w ' + self.testClass + '/' + self.testRunner + '; echo "ERROR: $?"')
        if DEBUG:
            print >> sys.stderr, "\nFinished test."
        errmsg = out.splitlines()[-1]
        m = re.match('ERROR: (\d+)', errmsg)
        if m:
            exitval = int(m.group(1))
            if exitval != 0:
                raise RuntimeError('Cannot start test on device: ' + out)
        else:
            raise RuntimeError('Unknown message')

    def forceStop(self):
        if DEBUG:
            print >> sys.stderr, "Cleaning up before start. Stopping '%s'" % self.pkg
        self.adbClient.shell('am force-stop ' + self.pkg)

class UiAutomatorHelper:
    PACKAGE = 'com.dtmilano.android.culebratester'
    TEST_CLASS = PACKAGE + '.test'
    TEST_RUNNER = 'com.dtmilano.android.uiautomatorhelper.UiAutomatorHelperTestRunner'

    def __init__(self, adbclient, adb=None, localport=9999, remoteport=9999, hostname='localhost'):
        if not REQUESTS_AVAILABLE:
            raise Exception('''Python Requests is needed for UiAutomatorHelper to work.

On Ubuntu install

   $ sudo apt-get install python-requests

On OSX install

   $ easy_install requests
''')

        self.adbClient = adbclient
        ''' The adb client (a.k.a. device) '''
        instrumentation = self.adbClient.shell('pm list instrumentation %s' % self.PACKAGE)
        if not instrumentation:
            raise RuntimeError('The target device does not contain the instrumentation for %s' % self.PACKAGE)
        if not re.match('instrumentation:%s/%s \(target=%s\)' % (self.TEST_CLASS, self.TEST_RUNNER, self.PACKAGE), instrumentation):
            raise RuntimeError('The instrumentation found for %s does not match the expected %s/%s' % (self.PACKAGE, self.TEST_CLASS, self.TEST_RUNNER))
        self.adb = self.__whichAdb(adb)
        ''' The adb command '''
        self.osName = platform.system()
        ''' The OS name. We sometimes need specific behavior. '''
        self.isDarwin = (self.osName == 'Darwin')
        ''' Is it Mac OSX? '''
        self.hostname = hostname
        ''' The hostname we are connecting to. '''
        if hostname in ['localhost', '127.0.0.1']:
            self.__redirectPort(localport, remoteport)
        self.__runTests()
        self.baseUrl = 'http://%s:%d' % (hostname, localport)
        try:
            self.session = self.__connectSession()
        except RuntimeError, ex:
            self.thread.forceStop()
            raise ex

    def __connectSession(self):
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: Acquiring lock"
        lock.acquire()
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: Lock acquired"
            print >> sys.stderr, "UiAutomatorHelper: Connecting session"
        session = requests.Session()
        if not session:
            raise RuntimeError("Cannot create session")
        tries = 10
        while tries > 0:
            time.sleep(0.5)
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: Attempting to connect to", self.baseUrl, '(tries=%s)' % tries
            try:
                response = session.head(self.baseUrl)
                if response.status_code == 200:
                    break
            except requests.exceptions.ConnectionError, ex:
                tries -= 1
        lock.release()
        if tries == 0:
            raise RuntimeError("Cannot connect to " + self.baseUrl)
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: HEAD", response
            print >> sys.stderr, "UiAutomatorHelper: Releasing lock"
        #lock.release()
        return session

    def __whichAdb(self, adb):
        if adb:
            if not os.access(adb, os.X_OK):
                raise Exception('adb="%s" is not executable' % adb)
        else:
            # Using adbclient we don't need adb executable yet (maybe it's needed if we want to
            # start adb if not running) or to redirect ports
            adb = obtainAdbPath()

        return adb

    def __redirectPort(self, localport, remoteport):
        self.localPort = localport
        self.remotePort = remoteport
        subprocess.check_call([self.adb, '-s', self.adbClient.serialno, 'forward', 'tcp:%d' % self.localPort,
                               'tcp:%d' % self.remotePort])

    def __runTests(self):
        if DEBUG:
            print >> sys.stderr, "__runTests: start"
        # We need a new AdbClient instance with timeout=None (means, no timeout) for the long running test service
        newAdbClient = AdbClient(self.adbClient.serialno, self.adbClient.hostname, self.adbClient.port, timeout=None)
        self.thread = RunTestsThread(adbClient=newAdbClient, testClass=self.TEST_CLASS, testRunner=self.TEST_RUNNER)
        if DEBUG:
            print >> sys.stderr, "__runTests: starting thread"
        self.thread.start()
        if DEBUG:
            print >> sys.stderr, "__runTests: end"

    def __httpCommand(self, url, params=None, method='GET'):
        if method == 'GET':
            if params:
                response = self.session.get(self.baseUrl + url, params=params)
            else:
                response = self.session.get(self.baseUrl + url)
        elif method == 'PUT':
            response = self.session.put(self.baseUrl + url, params=params)
        else:
            raise RuntimeError("method not supported: " + method)
        return response.content

    #
    # Device
    #
    def getDisplayRealSize(self):
        return self.__httpCommand('/Device/getDisplayRealSize')

    #
    # UiAutomatorHelper internal commands
    #
    def quit(self):
        try:
            self.__httpCommand('/UiAutomatorHelper/quit')
        except:
            pass
        self.session.close()

    #
    # UiDevice
    #
    def click(self, **kwargs):
        params = kwargs
        if not ((params.has_key('x') and params.has_key('y')) or params.has_key('oid')):
            raise RuntimeError('click: (x, y) or oid must have a value')
        if params.has_key('oid'):
            return self.__httpCommand('/UiObject2/%d/click' % params['oid'])
        else:
            return self.__httpCommand('/UiDevice/click', params)

    def dumpWindowHierarchy(self):
        dump = self.__httpCommand('/UiDevice/dumpWindowHierarchy').decode(encoding='UTF-8', errors='replace')
        if DEBUG:
            print >> sys.stderr, "DUMP: ", dump
        return dump

    def findObject(self, **kwargs):
        params = kwargs
        if not (params.has_key('resourceId') or params.has_key('bySelector')):
            raise RuntimeError('findObject: resourceId or bySelector must have a value')
        response = self.__httpCommand('/UiDevice/findObject', params)
        # { "status": "OK", "oid": 1, "className": "android.view.View"}
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: findObject: response=", response
        r = json.loads(response)
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: findObject: returning", int(r[u'oid'])
            return UiObject2(self, int(r[u'oid']))
        raise RuntimeError("Error: " + response)

    def longClick(self, **kwargs):
        params = kwargs
        if not ((params.has_key('x') and params.has_key('y')) or params.has_key('oid')):
            raise RuntimeError('longClick: (x, y) or oid must have a value')
        if params.has_key('oid'):
            return self.__httpCommand('/UiObject2/%d/longClick' % params['oid'])
        else:
            return self.__httpCommand('/UiDevice/longClick', params)

    def pressBack(self):
        return self.__httpCommand('/UiDevice/pressBack')

    def pressHome(self):
        return self.__httpCommand('/UiDevice/pressHome')

    def pressKeyCode(self, keyCode, metaState=0):
        params = {'keyCode': keyCode, 'metaState': metaState}
        return self.__httpCommand('/UiDevice/pressKeyCode', params)

    def pressRecentApps(self):
        return self.__httpCommand('/UiDevice/pressRecentApps')

    def swipe(self, startX=-1, startY=-1, endX=-1, endY=-1, steps=10, segments=[], segmentSteps=5):
        if startX != -1 and startY != -1:
            params = {'startX': startX, 'startY': startY, 'endX': endX, 'endY': endY, 'steps': steps}
        elif segments:
            params = {'segments': ','.join(str(p) for p in segments), "segmentSteps": segmentSteps}
        else:
            raise RuntimeError("Cannot determine method invocation from provided parameters. startX and startY or segments must be provided.")
        return self.__httpCommand('/UiDevice/swipe', params)

    def takeScreenshot(self, scale=1.0, quality=90):
        params = {'scale': scale, 'quality': quality}
        return self.__httpCommand('/UiDevice/takeScreenshot', params)

    #
    # UiObject
    #
    def setText(self, uiObject, text):
        params = {'text': text}
        return self.__httpCommand('/UiObject/0x%x/setText' % (uiObject.oid), params)

    #
    # UiObject2
    #
    def clickAndWait(self, uiObject2, eventCondition, timeout):
        params = {'eventCondition': eventCondition, 'timeout': timeout}
        return self.__httpCommand('/UiObject2/%d/clickAndWait' % (uiObject2.oid), params)

    def getText(self, uiObject2):
        response = self.__httpCommand('/UiObject2/%d/getText' % (uiObject2.oid), None)
        r = json.loads(response)
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: getText: returning", r[u'text']
            return r[u'text']
        raise RuntimeError("Error: " + response)

    #
    # UiScrollable
    #
    def uiScrollable(self, path, params):
        response = self.__httpCommand('/UiScrollable/' + path, params)
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: uiScrollable: response=", response
        r = None
        try:
            r = json.loads(response)
        except:
            print >> sys.stderr, "===================================="
            print >> sys.stderr, "Invalid JSON RESPONSE: ", response
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: uiScrollable: returning", int(r[u'oid'])
            return int(r[u'oid']), r
        if DEBUG:
            print >> sys.stderr, "RESPONSE: ", response
            print >> sus.stderr, "r=", r
        raise RuntimeError("Error: " + response)

class UiObject:
    def __init__(self, uiAutomatorHelper, oid):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.oid = oid

    def click(self):
        self.uiAutomatorHelper.click(oid=self.oid)

    def longClick(self):
        self.uiAutomatorHelper.longClick(oid=self.oid)

class UiObject2:
    def __init__(self, uiAutomatorHelper, oid):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.oid = oid

    def click(self):
        self.uiAutomatorHelper.click(oid=self.oid)

    def clickAndWait(self, eventCondition, timeout):
        self.uiAutomatorHelper.clickAndWait(uiObject2=self, eventCondition=eventCondition, timeout=timeout)

    def longClick(self):
        self.uiAutomatorHelper.longClick(oid=self.oid)

    def getText(self):
        return self.uiAutomatorHelper.getText(uiObject2=self)

    def setText(self, text):
        self.uiAutomatorHelper.setText(uiObject2=self, text=text)

class UiScrollable:
    def __init__(self, uiAutomatorHelper, uiSelector):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.uiSelector = uiSelector
        self.oid, self.response = self.__createUiScrollable()

    def __createUiScrollable(self):
        return self.uiAutomatorHelper.uiScrollable('new', {'uiSelector': self.uiSelector})

    def getChildByDescription(self, uiSelector, description, allowScrollSearch):
        oid, response = self.uiAutomatorHelper.uiScrollable(str(self.oid) + '/getChildByDescription', {'uiSelector': uiSelector, 'contentDescription': description, 'allowScrollSearch': allowScrollSearch})
        return UiObject(self.uiAutomatorHelper, oid)

    def getChildByText(self, uiSelector, text, allowScrollSearch):
        oid, response = self.uiAutomatorHelper.uiScrollable(str(self.oid) + '/getChildByText', {'uiSelector': uiSelector, 'text': text, 'allowScrollSearch': allowScrollSearch})
        return UiObject(self.uiAutomatorHelper, oid)
<commit_msg>Support for UiObject and UiObjext2 in getText<commit_after># -*- coding: utf-8 -*-
'''
Copyright (C) 2012-2015  Diego Torres Milano
Created on Feb 2, 2015

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@author: Diego Torres Milano
'''

__version__ = '12.1.2'

import os
import subprocess
import sys
import platform
import threading
import re
import json
try:
    import requests
    REQUESTS_AVAILABLE = True
except:
    REQUESTS_AVAILABLE = False
import time
from com.dtmilano.android.adb.adbclient import AdbClient
from com.dtmilano.android.common import obtainAdbPath

__author__ = 'diego'

DEBUG = False

lock = threading.Lock()

class RunTestsThread(threading.Thread):
    """
    Runs the instrumentation for the specified package in a new thread.
    """
    def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None, adbClient=None, testClass=None, testRunner=None):
        threading.Thread.__init__(self, group=group, target=target, name=name, verbose=verbose)
        self.adbClient = adbClient
        self.testClass = testClass
        self.testRunner = testRunner
        self.pkg = re.sub('\.test$', '', self.testClass)

    def run(self):
        if DEBUG:
            print >> sys.stderr, "RunTestsThread: Acquiring lock"
        lock.acquire()
        if DEBUG:
            print >> sys.stderr, "RunTestsThread: Lock acquired"
        self.forceStop()
        time.sleep(3)
        if DEBUG:
            print >> sys.stderr, "Starting test..."
            print >> sys.stderr, "RunTestsThread: Releasing lock"
        lock.release()
        out = self.adbClient.shell('am instrument -w ' + self.testClass + '/' + self.testRunner + '; echo "ERROR: $?"')
        if DEBUG:
            print >> sys.stderr, "\nFinished test."
        errmsg = out.splitlines()[-1]
        m = re.match('ERROR: (\d+)', errmsg)
        if m:
            exitval = int(m.group(1))
            if exitval != 0:
                raise RuntimeError('Cannot start test on device: ' + out)
        else:
            raise RuntimeError('Unknown message')

    def forceStop(self):
        if DEBUG:
            print >> sys.stderr, "Cleaning up before start. Stopping '%s'" % self.pkg
        self.adbClient.shell('am force-stop ' + self.pkg)

class UiAutomatorHelper:
    PACKAGE = 'com.dtmilano.android.culebratester'
    TEST_CLASS = PACKAGE + '.test'
    TEST_RUNNER = 'com.dtmilano.android.uiautomatorhelper.UiAutomatorHelperTestRunner'

    def __init__(self, adbclient, adb=None, localport=9999, remoteport=9999, hostname='localhost'):
        if not REQUESTS_AVAILABLE:
            raise Exception('''Python Requests is needed for UiAutomatorHelper to work.

On Ubuntu install

   $ sudo apt-get install python-requests

On OSX install

   $ easy_install requests
''')

        self.adbClient = adbclient
        ''' The adb client (a.k.a. device) '''
        instrumentation = self.adbClient.shell('pm list instrumentation %s' % self.PACKAGE)
        if not instrumentation:
            raise RuntimeError('The target device does not contain the instrumentation for %s' % self.PACKAGE)
        if not re.match('instrumentation:%s/%s \(target=%s\)' % (self.TEST_CLASS, self.TEST_RUNNER, self.PACKAGE), instrumentation):
            raise RuntimeError('The instrumentation found for %s does not match the expected %s/%s' % (self.PACKAGE, self.TEST_CLASS, self.TEST_RUNNER))
        self.adb = self.__whichAdb(adb)
        ''' The adb command '''
        self.osName = platform.system()
        ''' The OS name. We sometimes need specific behavior. '''
        self.isDarwin = (self.osName == 'Darwin')
        ''' Is it Mac OSX? '''
        self.hostname = hostname
        ''' The hostname we are connecting to. '''
        if hostname in ['localhost', '127.0.0.1']:
            self.__redirectPort(localport, remoteport)
        self.__runTests()
        self.baseUrl = 'http://%s:%d' % (hostname, localport)
        try:
            self.session = self.__connectSession()
        except RuntimeError, ex:
            self.thread.forceStop()
            raise ex

    def __connectSession(self):
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: Acquiring lock"
        lock.acquire()
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: Lock acquired"
            print >> sys.stderr, "UiAutomatorHelper: Connecting session"
        session = requests.Session()
        if not session:
            raise RuntimeError("Cannot create session")
        tries = 10
        while tries > 0:
            time.sleep(0.5)
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: Attempting to connect to", self.baseUrl, '(tries=%s)' % tries
            try:
                response = session.head(self.baseUrl)
                if response.status_code == 200:
                    break
            except requests.exceptions.ConnectionError, ex:
                tries -= 1
        lock.release()
        if tries == 0:
            raise RuntimeError("Cannot connect to " + self.baseUrl)
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: HEAD", response
            print >> sys.stderr, "UiAutomatorHelper: Releasing lock"
        #lock.release()
        return session

    def __whichAdb(self, adb):
        if adb:
            if not os.access(adb, os.X_OK):
                raise Exception('adb="%s" is not executable' % adb)
        else:
            # Using adbclient we don't need adb executable yet (maybe it's needed if we want to
            # start adb if not running) or to redirect ports
            adb = obtainAdbPath()

        return adb

    def __redirectPort(self, localport, remoteport):
        self.localPort = localport
        self.remotePort = remoteport
        subprocess.check_call([self.adb, '-s', self.adbClient.serialno, 'forward', 'tcp:%d' % self.localPort,
                               'tcp:%d' % self.remotePort])

    def __runTests(self):
        if DEBUG:
            print >> sys.stderr, "__runTests: start"
        # We need a new AdbClient instance with timeout=None (means, no timeout) for the long running test service
        newAdbClient = AdbClient(self.adbClient.serialno, self.adbClient.hostname, self.adbClient.port, timeout=None)
        self.thread = RunTestsThread(adbClient=newAdbClient, testClass=self.TEST_CLASS, testRunner=self.TEST_RUNNER)
        if DEBUG:
            print >> sys.stderr, "__runTests: starting thread"
        self.thread.start()
        if DEBUG:
            print >> sys.stderr, "__runTests: end"

    def __httpCommand(self, url, params=None, method='GET'):
        if method == 'GET':
            if params:
                response = self.session.get(self.baseUrl + url, params=params)
            else:
                response = self.session.get(self.baseUrl + url)
        elif method == 'PUT':
            response = self.session.put(self.baseUrl + url, params=params)
        else:
            raise RuntimeError("method not supported: " + method)
        return response.content

    #
    # Device
    #
    def getDisplayRealSize(self):
        return self.__httpCommand('/Device/getDisplayRealSize')

    #
    # UiAutomatorHelper internal commands
    #
    def quit(self):
        try:
            self.__httpCommand('/UiAutomatorHelper/quit')
        except:
            pass
        self.session.close()

    #
    # UiDevice
    #
    def click(self, **kwargs):
        params = kwargs
        if not ((params.has_key('x') and params.has_key('y')) or params.has_key('oid')):
            raise RuntimeError('click: (x, y) or oid must have a value')
        if params.has_key('oid'):
            return self.__httpCommand('/UiObject2/%d/click' % params['oid'])
        else:
            return self.__httpCommand('/UiDevice/click', params)

    def dumpWindowHierarchy(self):
        dump = self.__httpCommand('/UiDevice/dumpWindowHierarchy').decode(encoding='UTF-8', errors='replace')
        if DEBUG:
            print >> sys.stderr, "DUMP: ", dump
        return dump

    def findObject(self, **kwargs):
        params = kwargs
        if not (params.has_key('resourceId') or params.has_key('bySelector')):
            raise RuntimeError('findObject: resourceId or bySelector must have a value')
        response = self.__httpCommand('/UiDevice/findObject', params)
        # { "status": "OK", "oid": 1, "className": "android.view.View"}
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: findObject: response=", response
        r = json.loads(response)
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: findObject: returning", int(r[u'oid'])
            return UiObject2(self, int(r[u'oid']))
        raise RuntimeError("Error: " + response)

    def longClick(self, **kwargs):
        params = kwargs
        if not ((params.has_key('x') and params.has_key('y')) or params.has_key('oid')):
            raise RuntimeError('longClick: (x, y) or oid must have a value')
        if params.has_key('oid'):
            return self.__httpCommand('/UiObject2/%d/longClick' % params['oid'])
        else:
            return self.__httpCommand('/UiDevice/longClick', params)

    def pressBack(self):
        return self.__httpCommand('/UiDevice/pressBack')

    def pressHome(self):
        return self.__httpCommand('/UiDevice/pressHome')

    def pressKeyCode(self, keyCode, metaState=0):
        params = {'keyCode': keyCode, 'metaState': metaState}
        return self.__httpCommand('/UiDevice/pressKeyCode', params)

    def pressRecentApps(self):
        return self.__httpCommand('/UiDevice/pressRecentApps')

    def swipe(self, startX=-1, startY=-1, endX=-1, endY=-1, steps=10, segments=[], segmentSteps=5):
        if startX != -1 and startY != -1:
            params = {'startX': startX, 'startY': startY, 'endX': endX, 'endY': endY, 'steps': steps}
        elif segments:
            params = {'segments': ','.join(str(p) for p in segments), "segmentSteps": segmentSteps}
        else:
            raise RuntimeError("Cannot determine method invocation from provided parameters. startX and startY or segments must be provided.")
        return self.__httpCommand('/UiDevice/swipe', params)

    def takeScreenshot(self, scale=1.0, quality=90):
        params = {'scale': scale, 'quality': quality}
        return self.__httpCommand('/UiDevice/takeScreenshot', params)

    #
    # UiObject
    #
    def setText(self, uiObject, text):
        params = {'text': text}
        return self.__httpCommand('/UiObject/0x%x/setText' % (uiObject.oid), params)

    #
    # UiObject2
    #
    def clickAndWait(self, uiObject2, eventCondition, timeout):
        params = {'eventCondition': eventCondition, 'timeout': timeout}
        return self.__httpCommand('/UiObject2/%d/clickAndWait' % (uiObject2.oid), params)

    def getText(self, uiObject=None, uiObject2=None):
        if uiObject:
            path = '/UiObject/%d/getText' % (uiObject.oid)
        elif uiObject2:
            path = '/UiObject2/%d/getText' % (uiObject2.oid)
        else:
            raise ValueError("No uiObject or uiObject2 specified")
        response = self.__httpCommand(path, None)
        r = json.loads(response)
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: getText: returning", r[u'text']
            return r[u'text']
        raise RuntimeError("Error: " + response)

    #
    # UiScrollable
    #
    def uiScrollable(self, path, params):
        response = self.__httpCommand('/UiScrollable/' + path, params)
        if DEBUG:
            print >> sys.stderr, "UiAutomatorHelper: uiScrollable: response=", response
        r = None
        try:
            r = json.loads(response)
        except:
            print >> sys.stderr, "===================================="
            print >> sys.stderr, "Invalid JSON RESPONSE: ", response
        if r[u'status'] == 'OK':
            if DEBUG:
                print >> sys.stderr, "UiAutomatorHelper: uiScrollable: returning", int(r[u'oid'])
            return int(r[u'oid']), r
        if DEBUG:
            print >> sys.stderr, "RESPONSE: ", response
            print >> sus.stderr, "r=", r
        raise RuntimeError("Error: " + response)

class UiObject:
    def __init__(self, uiAutomatorHelper, oid):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.oid = oid

    def click(self):
        self.uiAutomatorHelper.click(oid=self.oid)

    def longClick(self):
        self.uiAutomatorHelper.longClick(oid=self.oid)

    def getText(self):
        return self.uiAutomatorHelper.getText(uiObject=self)

    def setText(self, text):
        self.uiAutomatorHelper.setText(uiObject=self, text=text)

class UiObject2:
    def __init__(self, uiAutomatorHelper, oid):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.oid = oid

    def click(self):
        self.uiAutomatorHelper.click(oid=self.oid)

    def clickAndWait(self, eventCondition, timeout):
        self.uiAutomatorHelper.clickAndWait(uiObject2=self, eventCondition=eventCondition, timeout=timeout)

    def longClick(self):
        self.uiAutomatorHelper.longClick(oid=self.oid)

    def getText(self):
        return self.uiAutomatorHelper.getText(uiObject2=self)

    def setText(self, text):
        self.uiAutomatorHelper.setText(uiObject2=self, text=text)

class UiScrollable:
    def __init__(self, uiAutomatorHelper, uiSelector):
        self.uiAutomatorHelper = uiAutomatorHelper
        self.uiSelector = uiSelector
        self.oid, self.response = self.__createUiScrollable()

    def __createUiScrollable(self):
        return self.uiAutomatorHelper.uiScrollable('new', {'uiSelector': self.uiSelector})

    def getChildByDescription(self, uiSelector, description, allowScrollSearch):
        oid, response = self.uiAutomatorHelper.uiScrollable(str(self.oid) + '/getChildByDescription', {'uiSelector': uiSelector, 'contentDescription': description, 'allowScrollSearch': allowScrollSearch})
        return UiObject(self.uiAutomatorHelper, oid)

    def getChildByText(self, uiSelector, text, allowScrollSearch):
        oid, response = self.uiAutomatorHelper.uiScrollable(str(self.oid) + '/getChildByText', {'uiSelector': uiSelector, 'text': text, 'allowScrollSearch': allowScrollSearch})
        return UiObject(self.uiAutomatorHelper, oid)
<|endoftext|>


<commit_before># Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interceptor that adds headers to outgoing requests."""

import collections

import grpc

class _ConcreteValue(grpc.Future):

    def __init__(self, result):
        self._result = result

    def cancel(self):
        return False

    def cancelled(self):
        return False

    def running(self):
        return False

    def done(self):
        return True

    def result(self, timeout=None):
        return self._result

    def exception(self, timeout=None):
        return None

    def traceback(self, timeout=None):
        return None

    def add_done_callback(self, fn):
        fn(self._result)

class DefaultValueClientInterceptor(grpc.UnaryUnaryClientInterceptor,
                                    grpc.StreamUnaryClientInterceptor):

    def __init__(self, value):
        self._default = _ConcreteValue(value)

    def _intercept_call(self, continuation, client_call_details,
                        request_or_iterator):
        response = continuation(client_call_details, request_or_iterator)
        return self._default if response.exception() else response

    def intercept_unary_unary(self, continuation, client_call_details, request):
        return self._intercept_call(continuation, client_call_details, request)

    def intercept_stream_unary(self, continuation, client_call_details,
                               request_iterator):
        return self._intercept_call(continuation, client_call_details,
                                    request_iterator)
<commit_msg>Remove unused import<commit_after># Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interceptor that adds headers to outgoing requests."""

import grpc

class _ConcreteValue(grpc.Future):

    def __init__(self, result):
        self._result = result

    def cancel(self):
        return False

    def cancelled(self):
        return False

    def running(self):
        return False

    def done(self):
        return True

    def result(self, timeout=None):
        return self._result

    def exception(self, timeout=None):
        return None

    def traceback(self, timeout=None):
        return None

    def add_done_callback(self, fn):
        fn(self._result)

class DefaultValueClientInterceptor(grpc.UnaryUnaryClientInterceptor,
                                    grpc.StreamUnaryClientInterceptor):

    def __init__(self, value):
        self._default = _ConcreteValue(value)

    def _intercept_call(self, continuation, client_call_details,
                        request_or_iterator):
        response = continuation(client_call_details, request_or_iterator)
        return self._default if response.exception() else response

    def intercept_unary_unary(self, continuation, client_call_details, request):
        return self._intercept_call(continuation, client_call_details, request)

    def intercept_stream_unary(self, continuation, client_call_details,
                               request_iterator):
        return self._intercept_call(continuation, client_call_details,
                                    request_iterator)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
from frappe.core.doctype.user_permission.user_permission import add_user_permissions
from frappe.permissions import has_user_permission

import frappe
import unittest

class TestUserPermission(unittest.TestCase):
	def setUp(self):
		frappe.db.sql("""DELETE FROM `tabUser Permission`
			WHERE `user` in (
				'test_bulk_creation_update@example.com',
				'test_user_perm1@example.com',
				'nested_doc_user@example.com')""")
		frappe.delete_doc_if_exists("DocType", "Person")
		frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`")

	def test_default_user_permission_validation(self):
		user = create_user('test_default_permission@example.com')
		param = get_params(user, 'User', user.name, is_default=1)
		add_user_permissions(param)
		#create a duplicate entry with default
		perm_user = create_user('test_user_perm@example.com')
		param = get_params(user, 'User', perm_user.name, is_default=1)
		self.assertRaises(frappe.ValidationError, add_user_permissions, param)

	def test_default_user_permission(self):
		frappe.set_user('Administrator')
		user = create_user('test_user_perm1@example.com', 'Website Manager')
		for category in ['general', 'public']:
			if not frappe.db.exists('Blog Category', category):
				frappe.get_doc({'doctype': 'Blog Category', 'title': category}).insert()

		param = get_params(user, 'Blog Category', 'general', is_default=1)
		add_user_permissions(param)

		param = get_params(user, 'Blog Category', 'public')
		add_user_permissions(param)

		frappe.set_user('test_user_perm1@example.com')
		doc = frappe.new_doc("Blog Post")

		self.assertEquals(doc.blog_category, 'general')
		frappe.set_user('Administrator')

	def test_apply_to_all(self):
		''' Create User permission for User having access to all applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)
		is_created = add_user_permissions(param)
		self.assertEquals(is_created, 1)

	def test_for_apply_to_all_on_update_from_apply_all(self):
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)

		# Initially create User Permission document with apply_to_all checked
		is_created = add_user_permissions(param)

		self.assertEquals(is_created, 1)
		is_created = add_user_permissions(param)

		# User Permission should not be changed
		self.assertEquals(is_created, 0)

	def test_for_applicable_on_update_from_apply_to_all(self):
		''' Update User Permission from all to some applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user,'User', user.name, applicable = ["Chat Room", "Chat Message"])

		# Initially create User Permission document with apply_to_all checked
		is_created = add_user_permissions(get_params(user, 'User', user.name))

		self.assertEquals(is_created, 1)

		is_created = add_user_permissions(param)
		frappe.db.commit()

		removed_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
		is_created_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Room"))
		is_created_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Message"))

		# Check that apply_to_all is removed
		self.assertIsNone(removed_apply_to_all)

		# Check that User Permissions for applicable is created
		self.assertIsNotNone(is_created_applicable_first)
		self.assertIsNotNone(is_created_applicable_second)
		self.assertEquals(is_created, 1)

	def test_for_apply_to_all_on_update_from_applicable(self):
		''' Update User Permission from some to all applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)

		# create User permissions that with applicable
		is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))

		self.assertEquals(is_created, 1)

		is_created = add_user_permissions(param)
		is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
		removed_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Room"))
		removed_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Message"))

		# To check that a User permission with apply_to_all exists
		self.assertIsNotNone(is_created_apply_to_all)

		# Check that all User Permission with applicable is removed
		self.assertIsNone(removed_applicable_first)
		self.assertIsNone(removed_applicable_second)
		self.assertEquals(is_created, 1)

	def test_user_perm_for_nested_doctype(self):
		"""Test if descendants' visibility is controlled for a nested DocType."""
		from frappe.core.doctype.doctype.test_doctype import new_doctype

		user = create_user("nested_doc_user@example.com", "Blogger")
		if not frappe.db.exists("DocType", "Person"):
			doc = new_doctype("Person",
				fields=[
					{
						"label": "Person Name",
						"fieldname": "person_name",
						"fieldtype": "Data"
					}
				], unique=0)
			doc.is_tree = 1
			doc.insert()

		parent_record = frappe.get_doc(
			{"doctype": "Person", "person_name": "Parent", "is_group": 1}
		).insert()

		child_record = frappe.get_doc(
			{"doctype": "Person", "person_name": "Child", "is_group": 0, "parent_person": parent_record.name}
		).insert()

		add_user_permissions(get_params(user, "Person", parent_record.name))

		# check if adding perm on a group record, makes child record visible
		self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name))
		self.assertTrue(has_user_permission(frappe.get_doc("Person", child_record.name), user.name))

		frappe.db.set_value("User Permission", {"allow": "Person", "for_value": parent_record.name}, "hide_descendants", 1)
		frappe.cache().delete_value("user_permissions")

		# check if adding perm on a group record with hide_descendants enabled,
		# hides child records
		self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name))
		self.assertFalse(has_user_permission(frappe.get_doc("Person", child_record.name), user.name))

def create_user(email, role="System Manager"):
	''' create user with role system manager '''
	if frappe.db.exists('User', email):
		return frappe.get_doc('User', email)
	else:
		user = frappe.new_doc('User')
		user.email = email
		user.first_name = email.split("@")[0]
		user.add_roles(role)
		return user

def get_params(user, doctype, docname, is_default=0, hide_descendants=0, applicable=None):
	''' Return param to insert '''
	param = {
		"user": user.name,
		"doctype":doctype,
		"docname":docname,
		"is_default": is_default,
		"apply_to_all_doctypes": 1,
		"applicable_doctypes": [],
		"hide_descendants": hide_descendants
	}
	if applicable:
		param.update({"apply_to_all_doctypes": 0})
		param.update({"applicable_doctypes": applicable})
	return param

def get_exists_param(user, applicable = None):
	''' param to check existing Document '''
	param = {
		"user": user.name,
		"allow": "User",
		"for_value": user.name,
	}
	if applicable:
		param.update({"applicable_for": applicable})
	else:
		param.update({"apply_to_all_doctypes": 1})
	return param
<commit_msg>test: allow adding multiple roles while creating user (#12616)<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
from frappe.core.doctype.user_permission.user_permission import add_user_permissions
from frappe.permissions import has_user_permission

import frappe
import unittest

class TestUserPermission(unittest.TestCase):
	def setUp(self):
		frappe.db.sql("""DELETE FROM `tabUser Permission`
			WHERE `user` in (
				'test_bulk_creation_update@example.com',
				'test_user_perm1@example.com',
				'nested_doc_user@example.com')""")
		frappe.delete_doc_if_exists("DocType", "Person")
		frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`")

	def test_default_user_permission_validation(self):
		user = create_user('test_default_permission@example.com')
		param = get_params(user, 'User', user.name, is_default=1)
		add_user_permissions(param)
		#create a duplicate entry with default
		perm_user = create_user('test_user_perm@example.com')
		param = get_params(user, 'User', perm_user.name, is_default=1)
		self.assertRaises(frappe.ValidationError, add_user_permissions, param)

	def test_default_user_permission(self):
		frappe.set_user('Administrator')
		user = create_user('test_user_perm1@example.com', 'Website Manager')
		for category in ['general', 'public']:
			if not frappe.db.exists('Blog Category', category):
				frappe.get_doc({'doctype': 'Blog Category', 'title': category}).insert()

		param = get_params(user, 'Blog Category', 'general', is_default=1)
		add_user_permissions(param)

		param = get_params(user, 'Blog Category', 'public')
		add_user_permissions(param)

		frappe.set_user('test_user_perm1@example.com')
		doc = frappe.new_doc("Blog Post")

		self.assertEquals(doc.blog_category, 'general')
		frappe.set_user('Administrator')

	def test_apply_to_all(self):
		''' Create User permission for User having access to all applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)
		is_created = add_user_permissions(param)
		self.assertEquals(is_created, 1)

	def test_for_apply_to_all_on_update_from_apply_all(self):
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)

		# Initially create User Permission document with apply_to_all checked
		is_created = add_user_permissions(param)

		self.assertEquals(is_created, 1)
		is_created = add_user_permissions(param)

		# User Permission should not be changed
		self.assertEquals(is_created, 0)

	def test_for_applicable_on_update_from_apply_to_all(self):
		''' Update User Permission from all to some applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user,'User', user.name, applicable = ["Chat Room", "Chat Message"])

		# Initially create User Permission document with apply_to_all checked
		is_created = add_user_permissions(get_params(user, 'User', user.name))

		self.assertEquals(is_created, 1)

		is_created = add_user_permissions(param)
		frappe.db.commit()

		removed_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
		is_created_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Room"))
		is_created_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Message"))

		# Check that apply_to_all is removed
		self.assertIsNone(removed_apply_to_all)

		# Check that User Permissions for applicable is created
		self.assertIsNotNone(is_created_applicable_first)
		self.assertIsNotNone(is_created_applicable_second)
		self.assertEquals(is_created, 1)

	def test_for_apply_to_all_on_update_from_applicable(self):
		''' Update User Permission from some to all applicable Doctypes'''
		user = create_user('test_bulk_creation_update@example.com')
		param = get_params(user, 'User', user.name)

		# create User permissions that with applicable
		is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))

		self.assertEquals(is_created, 1)

		is_created = add_user_permissions(param)
		is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
		removed_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Room"))
		removed_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Chat Message"))

		# To check that a User permission with apply_to_all exists
		self.assertIsNotNone(is_created_apply_to_all)

		# Check that all User Permission with applicable is removed
		self.assertIsNone(removed_applicable_first)
		self.assertIsNone(removed_applicable_second)
		self.assertEquals(is_created, 1)

	def test_user_perm_for_nested_doctype(self):
		"""Test if descendants' visibility is controlled for a nested DocType."""
		from frappe.core.doctype.doctype.test_doctype import new_doctype

		user = create_user("nested_doc_user@example.com", "Blogger")
		if not frappe.db.exists("DocType", "Person"):
			doc = new_doctype("Person",
				fields=[
					{
						"label": "Person Name",
						"fieldname": "person_name",
						"fieldtype": "Data"
					}
				], unique=0)
			doc.is_tree = 1
			doc.insert()

		parent_record = frappe.get_doc(
			{"doctype": "Person", "person_name": "Parent", "is_group": 1}
		).insert()

		child_record = frappe.get_doc(
			{"doctype": "Person", "person_name": "Child", "is_group": 0, "parent_person": parent_record.name}
		).insert()

		add_user_permissions(get_params(user, "Person", parent_record.name))

		# check if adding perm on a group record, makes child record visible
		self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name))
		self.assertTrue(has_user_permission(frappe.get_doc("Person", child_record.name), user.name))

		frappe.db.set_value("User Permission", {"allow": "Person", "for_value": parent_record.name}, "hide_descendants", 1)
		frappe.cache().delete_value("user_permissions")

		# check if adding perm on a group record with hide_descendants enabled,
		# hides child records
		self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name))
		self.assertFalse(has_user_permission(frappe.get_doc("Person", child_record.name), user.name))

def create_user(email, *roles):
	''' create user with role system manager '''
	if frappe.db.exists('User', email):
		return frappe.get_doc('User', email)

	user = frappe.new_doc('User')
	user.email = email
	user.first_name = email.split("@")[0]

	if not roles:
		roles = ('System Manager',)

	user.add_roles(*roles)
	return user

def get_params(user, doctype, docname, is_default=0, hide_descendants=0, applicable=None):
	''' Return param to insert '''
	param = {
		"user": user.name,
		"doctype":doctype,
		"docname":docname,
		"is_default": is_default,
		"apply_to_all_doctypes": 1,
		"applicable_doctypes": [],
		"hide_descendants": hide_descendants
	}
	if applicable:
		param.update({"apply_to_all_doctypes": 0})
		param.update({"applicable_doctypes": applicable})
	return param

def get_exists_param(user, applicable = None):
	''' param to check existing Document '''
	param = {
		"user": user.name,
		"allow": "User",
		"for_value": user.name,
	}
	if applicable:
		param.update({"applicable_for": applicable})
	else:
		param.update({"apply_to_all_doctypes": 1})
	return param
<|endoftext|>