<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download, unzip, clean and load the latest CAL-ACCESS database ZIP.
"""
import os
import logging
from sys import exit
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from django.core.files import File
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime
from clint.textui import progress
from django.utils.timezone import now
from django.core.management import call_command
from calaccess_raw.management import handle_command
from django.core.management.base import CommandError
from django.template.loader import render_to_string
from calaccess_raw.models.tracking import RawDataVersion
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.management.commands.extractcalaccessrawfiles import TestCommand as TestExtractCommand
from calaccess_raw import (
    get_download_directory,
    get_test_download_directory,
    get_model_list
)
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Download, unzip, clean and load the latest CAL-ACCESS database ZIP.
    """
    help = "Download, unzip, clean and load the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--keep-files",
            action="store_true",
            dest="keep_files",
            default=False,
            help="Keep zip, unzipped, TSV and CSV files"
        )
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Update or resume previous update without asking permission"
        )
        parser.add_argument(
            "--test",
            "--use-test-data",
            action="store_true",
            dest="test_data",
            default=False,
            help="Use sampled test data (skips download, clean a load)"
        )
        parser.add_argument(
            "-a",
            "--app-name",
            dest="app_name",
            default="calaccess_raw",
            help="Name of Django app with models into which data will "
                 "be imported (if other not calaccess_raw)"
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # set / compute any attributes that multiple class methods need
        self.app_name = options["app_name"]
        self.keep_files = options["keep_files"]
        self.test_mode = options['test_data']
        self.noinput = options['noinput']

        if self.test_mode:
            # and always keep files when running test data
            self.keep_files = True
            self.data_dir = get_test_download_directory()
            # need to set this app-wide because cleancalaccessrawfile
            #   also calls get_download_directory
            settings.CALACCESS_DOWNLOAD_DIR = self.data_dir
            self.noinput = True
        else:
            self.data_dir = get_download_directory()

        # set the download zip file path
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])
        # make the data dir if necessary
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        # Immediately check that the tsv directory exists when using test data,
        #   so we can stop immediately.
        if self.test_mode:
            if not os.path.exists(self.tsv_dir):
                raise CommandError("Data tsv directory does not exist "
                                   "at %s" % self.tsv_dir)
            elif self.verbosity:
                self.log("Using test data")

        # set up the dir for clean files
        self.csv_dir = os.path.join(self.data_dir, "csv/")
        os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir)

        # in test mode, get download metadata from local file
        if self.test_mode:
            with open(self.data_dir + "/sampled_version.txt", "r") as f:
                # get or create the RawDataVersion
                latest_version, created = RawDataVersion.objects.get_or_create(
                    release_datetime=f.readline(),
                    expected_size=f.readline()
                )
        # else request it
        else:
            download_metadata = self.get_download_metadata()
            logger.debug('ETag: %s' % download_metadata['etag'])
            logger.debug('Last-Modified: %s' % download_metadata['last-modified'])
            logger.debug('Content-Length: %s' % download_metadata['content-length'])
            # get or create the RawDataVersion
            latest_version, created = RawDataVersion.objects.get_or_create(
                release_datetime=download_metadata['last-modified'],
                expected_size=download_metadata['content-length']
            )
        # log if latest version is new
        if created:
            logger.info('New CAL-ACCESS version available.')

        previous_version = None
        since_previous_release = None
        can_resume_previous = False
        # if update to latest version has not yet started, check to see if previous
        #   version can resume
        if not latest_version.update_start_datetime:
            # get previous versions
            prev_downloaded_versions = RawDataVersion.objects.filter(
                download_start_datetime__isnull=False
            ).exclude(id=latest_version.id)
            # if there are any
            if prev_downloaded_versions:
                # get the last one started
                previous_version = prev_downloaded_versions.latest(
                    'download_start_datetime'
                )
                # for prompt, get time since previous release
                since_previous_release = naturaltime(
                    previous_version.release_datetime
                )
                # if the previous version finished downloading and didn't finish updating
                if (
                    previous_version.download_completed and
                    not previous_version.update_completed
                ):
                    # can resume if zip file has not been deleted
                    #   or if the raw files have been extracted
                    if (
                        os.path.exists(self.zip_path) or
                        previous_version.extract_completed
                    ):
                        can_resume_previous = True

        # if not accepting input
        if self.noinput:
            # skip if up-to-date
            if latest_version.update_completed:
                self.log('Your database is already up-to-date.')
                exit(0)
            else:
                # set to resume if update to latest is stalled or can resume previous
                self.resume = latest_version.update_stalled or can_resume_previous
        else:
            # set up prompt
            prompt_context = dict(
                latest_version=latest_version,
                previous_version=previous_version,
                can_resume_previous=can_resume_previous,
                since_previous_release=since_previous_release,
            )
            # render prompt string
            prompt = render_to_string(
                'calaccess_raw/updatecalaccessrawdata.txt',
                prompt_context,
            )
            # if the user can resume (latest or previous)
            if latest_version.update_stalled or can_resume_previous:
                # and initially confirms
                if self.confirm_proceed(prompt):
                    # go into resume mode
                    self.resume = True
                # if user does not initially confirm
                else:
                    self.resume = False
                    # if can resume latest (but doesn't) prompt restart
                    if latest_version.update_stalled:
                        confirm_restart = self.confirm_proceed(
                            'Do you want re-start your update?\n'
                        )
                    # if can resume previous (but doesn't) prompt for update to latest
                    elif can_resume_previous:
                        confirm_restart = self.confirm_proceed(
                            'Do you want to update to the latest version?\n'
                        )
                    # cancel if user doesn't confirm restart
                    if not confirm_restart:
                        raise CommandError("Update cancelled")
            # if can't resume
            else:
                self.resume = False
                # cancel if user doesn't initially confirm
                if not self.confirm_proceed(prompt):
                    raise CommandError("Update cancelled")

        # set to force restart if the user could have resumed but didn't
        force_restart = (
            (latest_version.update_stalled or can_resume_previous) and
            not self.resume
        )

        # set the version to be updated
        if not force_restart and can_resume_previous:
            self.version = previous_version
        else:
            self.version = latest_version

        # if not resuming, store the update start time
        if not self.resume:
            self.version.update_start_datetime = now()
        # either way, reset the finish time
        self.version.update_finish_datetime = None
        # save here in case the command doesn't finish
        self.version.save()

        # handle download
        if self.test_mode:
            pass
        # if the last download of the version completed and not forcing restart
        #   and the zip file is still there
        elif (
            self.version.download_completed and
            not force_restart and
            os.path.exists(self.zip_path)
        ):
            self.log('Already downloaded.')
        else:
            call_command(
                "downloadcalaccessrawdata",
                verbosity=self.verbosity,
                noinput=True,
                restart=force_restart,
            )
            if self.verbosity:
                self.duration()

        # handle file extraction
        if self.test_mode:
            handle_command(TestExtractCommand, verbosity=self.verbosity)
        # if the last extact of the version completed and not forcing restart
        elif self.version.extract_completed and not force_restart:
            self.log('Already extracted.')
        else:
            call_command(
                'extractcalaccessrawfiles',
                keep_files=self.keep_files
            )
            if self.verbosity:
                self.duration()

        # refresh the version (to get timestamp field values)
        self.version.refresh_from_db()

        self.clean()
        if self.verbosity:
            self.duration()

        self.load()
        if self.verbosity:
            self.duration()

        # store update finish time
        self.version.update_finish_datetime = now()
        # and save the RawDataVersion
        self.version.save()

        if self.verbosity:
            self.success("Done!")

    def clean(self):
        """
        Clean up the raw data files from the state so they are ready to get loaded into the database.
        """
        if self.verbosity:
            self.header("Cleaning data files")

        tsv_list = [
            f for f in os.listdir(self.tsv_dir) if '.TSV' in f.upper()
        ]

        if self.resume:
            # get finished clean command logs of last update
            prev_cleaned = [
                x.file_name + '.TSV'
                for x in self.version.files.filter(
                    clean_finish_datetime__isnull=False
                )
            ]
            self.log("{} files already cleaned.".format(len(prev_cleaned)))
            # remove these from tsv_list
            tsv_list = [x for x in tsv_list if x not in prev_cleaned]

        # Loop through all the files in the source directory
        if self.verbosity:
            tsv_list = progress.bar(tsv_list)
        for name in tsv_list:
            call_command(
                "cleancalaccessrawfile",
                name,
                verbosity=self.verbosity,
                keep_file=self.keep_files,
            )

        # if archive setting is enabled, zip up all of the csv and error logs
        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            # skip if resuming and the clean zip file is already saved
            if self.resume and self.version.clean_zip_archive:
                pass
            else:
                if self.verbosity:
                    self.header("Zipping cleaned files")
                # Remove previous zip file
                self.version.clean_zip_archive.delete()
                clean_zip_path = os.path.join(self.data_dir, 'clean.zip')

                # enable zipfile compression
                compression = ZIP_DEFLATED

                try:
                    zf = ZipFile(clean_zip_path, 'w', compression, allowZip64=True)
                except RuntimeError:
                    logger.error('Zip file cannot be compressed (check zlib module).')
                    compression = ZIP_STORED
                    zf = ZipFile(clean_zip_path, 'w', compression, allowZip64=True)

                # loop over and save files in csv dir
                for f in os.listdir(self.csv_dir):
                    csv_path = os.path.join(self.csv_dir, f)
                    zf.write(csv_path, f)

                # same for errors dir
                errors_dir = os.path.join(self.data_dir, 'log')
                for f in os.listdir(errors_dir):
                    error_path = os.path.join(errors_dir, f)
                    zf.write(error_path, f)

                # close the zip file
                zf.close()

                if not self.test_mode:
                    # save the clean zip size
                    self.version.clean_zip_size = os.path.getsize(clean_zip_path)
                    with open(clean_zip_path) as zf:
                        # Save the zip on the raw data version
                        self.version.clean_zip_archive.save(
                            os.path.basename(clean_zip_path), File(zf)
                        )

    def load(self):
        """
        Loads the cleaned up csv files into the database.
        """
        if self.verbosity:
            self.header("Loading data files")

        model_list = [
            x for x in get_model_list() if os.path.exists(x.objects.get_csv_path())
        ]

        if self.resume:
            # get finished load command logs of last update
            prev_loaded = [
                x.file_name
                for x in self.version.files.filter(
                    load_finish_datetime__isnull=False
                )
            ]
            self.log("{} models already loaded.".format(len(prev_loaded)))
            # remove these from model_list
            model_list = [x for x in model_list if x._meta.db_table not in prev_loaded]

        if self.verbosity:
            model_list = progress.bar(model_list)
        for model in model_list:
            call_command(
                "loadcalaccessrawfile",
                model.__name__,
                verbosity=self.verbosity,
                keep_file=self.keep_files,
                app_name=self.app_name,
            )
<commit_msg>Try download, re-try 5 mins later if CommandError raised<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download, unzip, clean and load the latest CAL-ACCESS database ZIP.
"""
import os
import logging
from sys import exit
from time import sleep
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from django.core.files import File
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime
from clint.textui import progress
from django.utils.timezone import now
from django.core.management import call_command
from calaccess_raw.management import handle_command
from django.core.management.base import CommandError
from django.template.loader import render_to_string
from calaccess_raw.models.tracking import RawDataVersion
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.management.commands.extractcalaccessrawfiles import TestCommand as TestExtractCommand
from calaccess_raw import (
    get_download_directory,
    get_test_download_directory,
    get_model_list
)
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Download, unzip, clean and load the latest CAL-ACCESS database ZIP.
    """
    help = "Download, unzip, clean and load the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--keep-files",
            action="store_true",
            dest="keep_files",
            default=False,
            help="Keep zip, unzipped, TSV and CSV files"
        )
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Update or resume previous update without asking permission"
        )
        parser.add_argument(
            "--test",
            "--use-test-data",
            action="store_true",
            dest="test_data",
            default=False,
            help="Use sampled test data (skips download, clean a load)"
        )
        parser.add_argument(
            "-a",
            "--app-name",
            dest="app_name",
            default="calaccess_raw",
            help="Name of Django app with models into which data will "
                 "be imported (if other not calaccess_raw)"
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # set / compute any attributes that multiple class methods need
        self.app_name = options["app_name"]
        self.keep_files = options["keep_files"]
        self.test_mode = options['test_data']
        self.noinput = options['noinput']

        if self.test_mode:
            # and always keep files when running test data
            self.keep_files = True
            self.data_dir = get_test_download_directory()
            # need to set this app-wide because cleancalaccessrawfile
            #   also calls get_download_directory
            settings.CALACCESS_DOWNLOAD_DIR = self.data_dir
            self.noinput = True
        else:
            self.data_dir = get_download_directory()

        # set the download zip file path
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])
        # make the data dir if necessary
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        # Immediately check that the tsv directory exists when using test data,
        #   so we can stop immediately.
        if self.test_mode:
            if not os.path.exists(self.tsv_dir):
                raise CommandError("Data tsv directory does not exist "
                                   "at %s" % self.tsv_dir)
            elif self.verbosity:
                self.log("Using test data")

        # set up the dir for clean files
        self.csv_dir = os.path.join(self.data_dir, "csv/")
        os.path.exists(self.csv_dir) or os.makedirs(self.csv_dir)

        # in test mode, get download metadata from local file
        if self.test_mode:
            with open(self.data_dir + "/sampled_version.txt", "r") as f:
                # get or create the RawDataVersion
                latest_version, created = RawDataVersion.objects.get_or_create(
                    release_datetime=f.readline(),
                    expected_size=f.readline()
                )
        # else request it
        else:
            download_metadata = self.get_download_metadata()
            logger.debug('ETag: %s' % download_metadata['etag'])
            logger.debug('Last-Modified: %s' % download_metadata['last-modified'])
            logger.debug('Content-Length: %s' % download_metadata['content-length'])
            # get or create the RawDataVersion
            latest_version, created = RawDataVersion.objects.get_or_create(
                release_datetime=download_metadata['last-modified'],
                expected_size=download_metadata['content-length']
            )
        # log if latest version is new
        if created:
            logger.info('New CAL-ACCESS version available.')

        previous_version = None
        since_previous_release = None
        can_resume_previous = False
        # if update to latest version has not yet started, check to see if previous
        #   version can resume
        if not latest_version.update_start_datetime:
            # get previous versions
            prev_downloaded_versions = RawDataVersion.objects.filter(
                download_start_datetime__isnull=False
            ).exclude(id=latest_version.id)
            # if there are any
            if prev_downloaded_versions:
                # get the last one started
                previous_version = prev_downloaded_versions.latest(
                    'download_start_datetime'
                )
                # for prompt, get time since previous release
                since_previous_release = naturaltime(
                    previous_version.release_datetime
                )
                # if the previous version finished downloading and didn't finish updating
                if (
                    previous_version.download_completed and
                    not previous_version.update_completed
                ):
                    # can resume if zip file has not been deleted
                    #   or if the raw files have been extracted
                    if (
                        os.path.exists(self.zip_path) or
                        previous_version.extract_completed
                    ):
                        can_resume_previous = True

        # if not accepting input
        if self.noinput:
            # skip if up-to-date
            if latest_version.update_completed:
                self.log('Your database is already up-to-date.')
                exit(0)
            else:
                # set to resume if update to latest is stalled or can resume previous
                self.resume = latest_version.update_stalled or can_resume_previous
        else:
            # set up prompt
            prompt_context = dict(
                latest_version=latest_version,
                previous_version=previous_version,
                can_resume_previous=can_resume_previous,
                since_previous_release=since_previous_release,
            )
            # render prompt string
            prompt = render_to_string(
                'calaccess_raw/updatecalaccessrawdata.txt',
                prompt_context,
            )
            # if the user can resume (latest or previous)
            if latest_version.update_stalled or can_resume_previous:
                # and initially confirms
                if self.confirm_proceed(prompt):
                    # go into resume mode
                    self.resume = True
                # if user does not initially confirm
                else:
                    self.resume = False
                    # if can resume latest (but doesn't) prompt restart
                    if latest_version.update_stalled:
                        confirm_restart = self.confirm_proceed(
                            'Do you want re-start your update?\n'
                        )
                    # if can resume previous (but doesn't) prompt for update to latest
                    elif can_resume_previous:
                        confirm_restart = self.confirm_proceed(
                            'Do you want to update to the latest version?\n'
                        )
                    # cancel if user doesn't confirm restart
                    if not confirm_restart:
                        raise CommandError("Update cancelled")
            # if can't resume
            else:
                self.resume = False
                # cancel if user doesn't initially confirm
                if not self.confirm_proceed(prompt):
                    raise CommandError("Update cancelled")

        # set to force restart if the user could have resumed but didn't
        force_restart = (
            (latest_version.update_stalled or can_resume_previous) and
            not self.resume
        )

        # set the version to be updated
        if not force_restart and can_resume_previous:
            self.version = previous_version
        else:
            self.version = latest_version

        # if not resuming, store the update start time
        if not self.resume:
            self.version.update_start_datetime = now()
        # either way, reset the finish time
        self.version.update_finish_datetime = None
        # save here in case the command doesn't finish
        self.version.save()

        # handle download
        if self.test_mode:
            pass
        # if the last download of the version completed and not forcing restart
        #   and the zip file is still there
        elif (
            self.version.download_completed and
            not force_restart and
            os.path.exists(self.zip_path)
        ):
            self.log('Already downloaded.')
        else:
            try:
                call_command(
                    "downloadcalaccessrawdata",
                    verbosity=self.verbosity,
                    noinput=True,
                    restart=force_restart,
                )
            except CommandError as e:
                # if the expected and actual zip size are not the same
                if 'expected' in e.message.lower():
                    logger.debug('Waiting five minutes before re-trying')
                    # wait five minutes
                    sleep(300)
                    # then try again
                    call_command(
                        "downloadcalaccessrawdata",
                        verbosity=self.verbosity,
                        noinput=True,
                        # force a restart on second try
                        restart=True,
                    )

            if self.verbosity:
                self.duration()

        # handle file extraction
        if self.test_mode:
            handle_command(TestExtractCommand, verbosity=self.verbosity)
        # if the last extact of the version completed and not forcing restart
        elif self.version.extract_completed and not force_restart:
            self.log('Already extracted.')
        else:
            call_command(
                'extractcalaccessrawfiles',
                keep_files=self.keep_files
            )
            if self.verbosity:
                self.duration()

        # refresh the version (to get timestamp field values)
        self.version.refresh_from_db()

        self.clean()
        if self.verbosity:
            self.duration()

        self.load()
        if self.verbosity:
            self.duration()

        # store update finish time
        self.version.update_finish_datetime = now()
        # and save the RawDataVersion
        self.version.save()

        if self.verbosity:
            self.success("Done!")

    def clean(self):
        """
        Clean up the raw data files from the state so they are ready to get loaded into the database.
        """
        if self.verbosity:
            self.header("Cleaning data files")

        tsv_list = [
            f for f in os.listdir(self.tsv_dir) if '.TSV' in f.upper()
        ]

        if self.resume:
            # get finished clean command logs of last update
            prev_cleaned = [
                x.file_name + '.TSV'
                for x in self.version.files.filter(
                    clean_finish_datetime__isnull=False
                )
            ]
            self.log("{} files already cleaned.".format(len(prev_cleaned)))
            # remove these from tsv_list
            tsv_list = [x for x in tsv_list if x not in prev_cleaned]

        # Loop through all the files in the source directory
        if self.verbosity:
            tsv_list = progress.bar(tsv_list)
        for name in tsv_list:
            call_command(
                "cleancalaccessrawfile",
                name,
                verbosity=self.verbosity,
                keep_file=self.keep_files,
            )

        # if archive setting is enabled, zip up all of the csv and error logs
        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            # skip if resuming and the clean zip file is already saved
            if self.resume and self.version.clean_zip_archive:
                pass
            else:
                if self.verbosity:
                    self.header("Zipping cleaned files")
                # Remove previous zip file
                self.version.clean_zip_archive.delete()
                clean_zip_path = os.path.join(self.data_dir, 'clean.zip')

                # enable zipfile compression
                compression = ZIP_DEFLATED

                try:
                    zf = ZipFile(clean_zip_path, 'w', compression, allowZip64=True)
                except RuntimeError:
                    logger.error('Zip file cannot be compressed (check zlib module).')
                    compression = ZIP_STORED
                    zf = ZipFile(clean_zip_path, 'w', compression, allowZip64=True)

                # loop over and save files in csv dir
                for f in os.listdir(self.csv_dir):
                    csv_path = os.path.join(self.csv_dir, f)
                    zf.write(csv_path, f)

                # same for errors dir
                errors_dir = os.path.join(self.data_dir, 'log')
                for f in os.listdir(errors_dir):
                    error_path = os.path.join(errors_dir, f)
                    zf.write(error_path, f)

                # close the zip file
                zf.close()

                if not self.test_mode:
                    # save the clean zip size
                    self.version.clean_zip_size = os.path.getsize(clean_zip_path)
                    with open(clean_zip_path) as zf:
                        # Save the zip on the raw data version
                        self.version.clean_zip_archive.save(
                            os.path.basename(clean_zip_path), File(zf)
                        )

    def load(self):
        """
        Loads the cleaned up csv files into the database.
        """
        if self.verbosity:
            self.header("Loading data files")

        model_list = [
            x for x in get_model_list() if os.path.exists(x.objects.get_csv_path())
        ]

        if self.resume:
            # get finished load command logs of last update
            prev_loaded = [
                x.file_name
                for x in self.version.files.filter(
                    load_finish_datetime__isnull=False
                )
            ]
            self.log("{} models already loaded.".format(len(prev_loaded)))
            # remove these from model_list
            model_list = [x for x in model_list if x._meta.db_table not in prev_loaded]

        if self.verbosity:
            model_list = progress.bar(model_list)
        for model in model_list:
            call_command(
                "loadcalaccessrawfile",
                model.__name__,
                verbosity=self.verbosity,
                keep_file=self.keep_files,
                app_name=self.app_name,
            )
<|endoftext|>


<commit_before>from django.db.models.loading import get_model
from django.core.management.base import LabelCommand

class Command(LabelCommand):
    help = 'Compare the number of records in a model against its source CSV'
    args = '<model name>'

    def handle_label(self, label, **options):
        print "- Verifying %s" % label

        # Get the model
        model = get_model("calaccess_raw", label)

        # Get the db total
        cnt = model.objects.count()

        # Get the CSV total
        csv_path = model.objects.get_csv_path()
        infile = open(csv_path)
        csv_record_cnt = len(infile.readlines()) - 1
        infile.close()

        # Report back on how we did
        if cnt == csv_record_cnt:
            print "-- Record counts match"
        else:
            print '-- Records don\'t match. Table: %s\tCSV: %s' % (
                cnt,
                csv_record_cnt,
            )
<commit_msg>Colors in verify command for #57.<commit_after>from django.db.models.loading import get_model
from django.core.management.base import LabelCommand
from calaccess_raw.management.commands import CalAccessCommand

class Command(CalAccessCommand, LabelCommand):
    help = 'Compare the number of records in a model against its source CSV'
    args = '<model name>'

    def handle_label(self, label, **options):
        self.log("Verifying %s" % label)

        # Get the model
        model = get_model("calaccess_raw", label)

        # Get the db total
        cnt = model.objects.count()

        # Get the CSV total
        csv_path = model.objects.get_csv_path()
        infile = open(csv_path)
        csv_record_cnt = len(infile.readlines()) - 1
        infile.close()

        # Report back on how we did
        if cnt == csv_record_cnt:
            self.success("-- Table record count matches CSV")
        else:
            self.failure('-- Table Record count doesn\'t match CSV. Table: %s\tCSV: %s' % (
                cnt,
                csv_record_cnt,
            ))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import
import llvm.core
from .intrinsic import IntrinsicLibrary
from .numba_intrinsic import is_numba_intrinsic

__all__ = []
all = {}

def _import_all():
    global __all__
    mods = ['math_intrinsic',
            'string_intrinsic']
    for k in mods:
        mod = __import__(__name__ + '.' + k, fromlist=['__all__'])
        __all__.extend(mod.__all__)
        for k in mod.__all__:
            all[k] = globals()[k] = getattr(mod, k)

_import_all()

def default_intrinsic_library(context):
    '''Build an intrinsic library with a default set of external functions.

    context --- numba context

    TODO: It is possible to cache the default intrinsic library as a bitcode
    file on disk so that we don't build it every time.
    '''
    intrlib = IntrinsicLibrary(context)
    # install intrinsics
    for fncls in all.itervalues():
        intrlib.add(fncls)
    return intrlib

<commit_msg>Fix old import of math intrinsics<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import
import llvm.core
from .intrinsic import IntrinsicLibrary
from .numba_intrinsic import is_numba_intrinsic

__all__ = []
all = {}

def _import_all():
    global __all__
    mods = ['string_intrinsic']
    for k in mods:
        mod = __import__(__name__ + '.' + k, fromlist=['__all__'])
        __all__.extend(mod.__all__)
        for k in mod.__all__:
            all[k] = globals()[k] = getattr(mod, k)

_import_all()

def default_intrinsic_library(context):
    '''Build an intrinsic library with a default set of external functions.

    context --- numba context

    TODO: It is possible to cache the default intrinsic library as a bitcode
    file on disk so that we don't build it every time.
    '''
    intrlib = IntrinsicLibrary(context)
    # install intrinsics
    for fncls in all.itervalues():
        intrlib.add(fncls)
    return intrlib

<|endoftext|>


<commit_before>from numba.nodes import *

class ClosureNode(Node):
    """
    Inner functions or closures.

    When coerced to an object, a wrapper PyMethodDef gets created, and at
    call time a function is dynamically created with the closure scope.

        func_def:
            AST FunctionDef of the function
        closure_type:
            numba.typesystem.ClosureType
        outer_py_func:
            Outer Python function (or None!)
    """

    _fields = []

    def __init__(self, func_def, closure_type, outer_py_func, **kwargs):
        super(ClosureNode, self).__init__(**kwargs)
        self.func_def = func_def
        self.type = closure_type
        self.outer_py_func = outer_py_func
        self.name = self.func_def.name

        self.need_numba_func = getattr(func_def, 'need_closure_wrapper', True)
        self.lfunc = None
        self.wrapper_func = None
        self.wrapper_lfunc = None

        # ast and symtab after type inference
        self.type_inferred_ast = None
        self.symtab = None

        # The Python extension type that must be instantiated to hold cellvars
        # self.scope_type = None
        self.ext_type = None
        self.need_closure_scope = False

    def make_pyfunc(self):
        d = self.outer_py_func.func_globals
#        argnames = tuple(arg.id for arg in self.func_def.args.args)
#        dummy_func_string = """
#def __numba_closure_func(%s):
#    pass
#        """ % ", ".join(argnames)
#        exec dummy_func_string in d, d

        # Something set a pure and original, unmodified, AST, use that
        # instead and reset it after the compile. This is a HACK
        func_body = self.func_def.body
        if hasattr(self.func_def, 'pure_ast_body'):
            self.func_def.body = self.func_def.pure_ast_body

        name = self.func_def.name
        self.func_def.name = '__numba_closure_func'
        ast_mod = ast.Module(body=[self.func_def])
        numba.functions.fix_ast_lineno(ast_mod)
        c = compile(ast_mod, '<string>', 'exec')
        exec c in d, d
        self.func_def.name = name

        self.py_func = d['__numba_closure_func']
        self.py_func.live_objects = []
        self.py_func.__module__ = self.outer_py_func.__module__
        self.py_func.__name__ = name

        if hasattr(self.func_def, 'pure_ast_body'):
            self.func_def.body = func_body

class InstantiateClosureScope(Node):

    _fields = ['outer_scope']

    def __init__(self, func_def, scope_ext_type, scope_type, outer_scope, **kwargs):
        super(InstantiateClosureScope, self).__init__(**kwargs)
        self.func_def = func_def
        self.scope_type = scope_type
        self.ext_type = scope_ext_type
        self.outer_scope = outer_scope
        self.type = scope_type

class ClosureScopeLoadNode(Node):
    "Load the closure scope for the function or NULL"

    type = void.pointer()

class ClosureCallNode(NativeCallNode):
    """
    Call to closure or inner function.
    """

    def __init__(self, closure_type, call_node, **kwargs):
        self.call_node = call_node
        args, keywords = call_node.args, call_node.keywords
        args = args + self._resolve_keywords(closure_type, args, keywords)
        super(ClosureCallNode, self).__init__(closure_type.signature, args,
                                              llvm_func=None, **kwargs)
        self.closure_type = closure_type

    def _resolve_keywords(self, closure_type, args, keywords):
        "Map keyword arguments to positional arguments"
        func_def = closure_type.closure.func_def
        argnames = [name.id for name in func_def.args.args]

        expected = len(argnames) - len(args)
        if len(keywords) != expected:
            raise error.NumbaError(
                    self.call_node,
                    "Expected %d arguments, got %d" % (len(argnames),
                                                       len(args) + len(keywords)))

        argpositions = dict(zip(argnames, range(len(argnames))))
        positional = [None] * (len(argnames) - len(args))

        for keyword in keywords:
            argname = keyword.arg
            pos = argpositions.get(argname, None)
            if pos is None:
                raise error.NumbaError(
                        keyword, "Not a valid keyword argument name: %s" % argname)
            elif pos < len(args):
                raise error.NumbaError(
                        keyword, "Got multiple values for positional "
                                 "argument %r" % argname)
            else:
                positional[pos] = keyword.value

        return positional
<commit_msg>Add lfunc_pointer to ClosureNode<commit_after>from numba.nodes import *

class ClosureNode(Node):
    """
    Inner functions or closures.

    When coerced to an object, a wrapper PyMethodDef gets created, and at
    call time a function is dynamically created with the closure scope.

        func_def:
            AST FunctionDef of the function
        closure_type:
            numba.typesystem.ClosureType
        outer_py_func:
            Outer Python function (or None!)
    """

    _fields = []

    def __init__(self, func_def, closure_type, outer_py_func, **kwargs):
        super(ClosureNode, self).__init__(**kwargs)
        self.func_def = func_def
        self.type = closure_type
        self.outer_py_func = outer_py_func
        self.name = self.func_def.name

        self.need_numba_func = getattr(func_def, 'need_closure_wrapper', True)
        self.lfunc = None
        self.wrapper_func = None
        self.wrapper_lfunc = None
        self.lfunc_pointer = None

        # ast and symtab after type inference
        self.type_inferred_ast = None
        self.symtab = None

        # The Python extension type that must be instantiated to hold cellvars
        # self.scope_type = None
        self.ext_type = None
        self.need_closure_scope = False

    def make_pyfunc(self):
        d = self.outer_py_func.func_globals
#        argnames = tuple(arg.id for arg in self.func_def.args.args)
#        dummy_func_string = """
#def __numba_closure_func(%s):
#    pass
#        """ % ", ".join(argnames)
#        exec dummy_func_string in d, d

        # Something set a pure and original, unmodified, AST, use that
        # instead and reset it after the compile. This is a HACK
        func_body = self.func_def.body
        if hasattr(self.func_def, 'pure_ast_body'):
            self.func_def.body = self.func_def.pure_ast_body

        name = self.func_def.name
        self.func_def.name = '__numba_closure_func'
        ast_mod = ast.Module(body=[self.func_def])
        numba.functions.fix_ast_lineno(ast_mod)
        c = compile(ast_mod, '<string>', 'exec')
        exec c in d, d
        self.func_def.name = name

        self.py_func = d['__numba_closure_func']
        self.py_func.live_objects = []
        self.py_func.__module__ = self.outer_py_func.__module__
        self.py_func.__name__ = name

        if hasattr(self.func_def, 'pure_ast_body'):
            self.func_def.body = func_body

class InstantiateClosureScope(Node):

    _fields = ['outer_scope']

    def __init__(self, func_def, scope_ext_type, scope_type, outer_scope, **kwargs):
        super(InstantiateClosureScope, self).__init__(**kwargs)
        self.func_def = func_def
        self.scope_type = scope_type
        self.ext_type = scope_ext_type
        self.outer_scope = outer_scope
        self.type = scope_type

class ClosureScopeLoadNode(Node):
    "Load the closure scope for the function or NULL"

    type = void.pointer()

class ClosureCallNode(NativeCallNode):
    """
    Call to closure or inner function.
    """

    def __init__(self, closure_type, call_node, **kwargs):
        self.call_node = call_node
        args, keywords = call_node.args, call_node.keywords
        args = args + self._resolve_keywords(closure_type, args, keywords)
        super(ClosureCallNode, self).__init__(closure_type.signature, args,
                                              llvm_func=None, **kwargs)
        self.closure_type = closure_type

    def _resolve_keywords(self, closure_type, args, keywords):
        "Map keyword arguments to positional arguments"
        func_def = closure_type.closure.func_def
        argnames = [name.id for name in func_def.args.args]

        expected = len(argnames) - len(args)
        if len(keywords) != expected:
            raise error.NumbaError(
                    self.call_node,
                    "Expected %d arguments, got %d" % (len(argnames),
                                                       len(args) + len(keywords)))

        argpositions = dict(zip(argnames, range(len(argnames))))
        positional = [None] * (len(argnames) - len(args))

        for keyword in keywords:
            argname = keyword.arg
            pos = argpositions.get(argname, None)
            if pos is None:
                raise error.NumbaError(
                        keyword, "Not a valid keyword argument name: %s" % argname)
            elif pos < len(args):
                raise error.NumbaError(
                        keyword, "Got multiple values for positional "
                                 "argument %r" % argname)
            else:
                positional[pos] = keyword.value

        return positional
<|endoftext|>


<commit_before>import os
import numpy as np
import ctypes

from numba import *
import numba

@autojit(backend='ast')
def cast_int():
    value = 1.7
    return int32(value)

@autojit(backend='ast')
def cast_complex():
    value = 1.2
    return complex128(value)

@autojit(backend='ast')
def cast_float():
    value = 5
    return float_(value)

@autojit(backend='ast')
def cast_object(dst_type):
    value = np.arange(10, dtype=np.double)
    return dst_type(value)

@autojit(backend='ast')
def cast_as_numba_type_attribute():
    value = 4.4
    return numba.int32(value)

def test_casts():
#    assert cast_int() == 1
#    assert cast_complex() == 1.2 + 0j
#    assert cast_float() == 5.0
#    assert np.all(cast_object(double[:]) == np.arange(10))
    assert cast_as_numba_type_attribute() == 4

if __name__ == "__main__":
    test_casts()<commit_msg>Update test<commit_after>import os
import numpy as np
import ctypes

from numba import *
import numba

@autojit(backend='ast')
def cast_int():
    value = 1.7
    return int32(value)

@autojit(backend='ast')
def cast_complex():
    value = 1.2
    return complex128(value)

@autojit(backend='ast')
def cast_float():
    value = 5
    return float_(value)

@autojit(backend='ast')
def cast_object(dst_type):
    value = np.arange(10, dtype=np.double)
    return dst_type(value)

@autojit(backend='ast')
def cast_as_numba_type_attribute():
    value = 4.4
    return numba.int32(value)

def test_casts():
    assert cast_int() == 1
    assert cast_complex() == 1.2 + 0j
    assert cast_float() == 5.0
    assert np.all(cast_object(double[:]) == np.arange(10))
    assert cast_as_numba_type_attribute() == 4

if __name__ == "__main__":
    test_casts()<|endoftext|>


<commit_before># future
from __future__ import annotations

# stdlib
import operator
from typing import Any
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

# third party
import numpy as np
import torch as th

# relative
from ... import lib
from ...ast.klass import pointerize_args_and_kwargs
from ...core.common.serde.recursive import RecursiveSerde
from ...util import inherit_tags
from ..common.serde.serializable import bind_protobuf
from ..common.uid import UID
from ..node.abstract.node import AbstractNodeClient
from ..node.common.action.run_class_method_action import RunClassMethodAction
from ..pointer.pointer import Pointer
from .ancestors import AutogradTensorAncestor
from .ancestors import PhiTensorAncestor
from .fixed_precision_tensor_ancestor import FixedPrecisionTensorAncestor
from .passthrough import PassthroughTensor  # type: ignore
from .smpc.mpc_tensor import MPCTensor

class TensorPointer(Pointer):

    # Must set these at class init time despite
    # the fact that klass.Class tries to override them (unsuccessfully)
    __name__ = "TensorPointer"
    __module__ = "syft.core.tensor.tensor"

    def __init__(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
        public_shape: Optional[Tuple[int, ...]] = None,
    ):

        super().__init__(
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

        self.public_shape = public_shape

    def share(self, *parties: Tuple[AbstractNodeClient, ...]) -> MPCTensor:

        parties = tuple(list(parties) + [self.client])

        self_mpc = MPCTensor(secret=self, shape=self.public_shape, parties=parties)

        return self_mpc

    def _apply_tensor_op(self, other: Any, op_str: str) -> Any:
        # we want to get the return type which matches the attr_path_and_name
        # so we ask lib_ast for the return type name that matches out
        # attr_path_and_name and then use that to get the actual pointer klass
        # then set the result to that pointer klass

        attr_path_and_name = "syft.core.tensor.tensor.Tensor."
        attr_path_and_name += "__" + op_str + "__"

        result = TensorPointer(client=self.client)

        # QUESTION can the id_at_location be None?
        result_id_at_location = getattr(result, "id_at_location", None)

        if result_id_at_location is not None:
            # first downcast anything primitive which is not already PyPrimitive
            (
                downcast_args,
                downcast_kwargs,
            ) = lib.python.util.downcast_args_and_kwargs(args=[other], kwargs={})

            # then we convert anything which isnt a pointer into a pointer
            pointer_args, pointer_kwargs = pointerize_args_and_kwargs(
                args=downcast_args,
                kwargs=downcast_kwargs,
                client=self.client,
                gc_enabled=False,
            )

            cmd = RunClassMethodAction(
                path=attr_path_and_name,
                _self=self,
                args=pointer_args,
                kwargs=pointer_kwargs,
                id_at_location=result_id_at_location,
                address=self.client.address,
            )
            self.client.send_immediate_msg_without_reply(msg=cmd)

        inherit_tags(
            attr_path_and_name=attr_path_and_name,
            result=result,
            self_obj=self,
            args=[other],
            kwargs={},
        )

        result_public_shape = None

        op = getattr(operator, op_str)

        if self.public_shape is not None and other.public_shape is not None:
            result_public_shape = (
                op(np.empty(self.public_shape), np.empty(other.public_shape))
            ).shape

        result.public_shape = result_public_shape

        return result

    @staticmethod
    def _apply_op(
        self: TensorPointer,
        other: Union[TensorPointer, MPCTensor, int, float, np.ndarray],
        op_str: str,
    ) -> Tuple[MPCTensor, Union[MPCTensor, int, float, np.ndarray]]:
        """Performs the operation based on op_str

        Args:
            other (Union[TensorPointer,MPCTensor,int,float,np.ndarray]): second operand.

        Returns:
            Tuple[MPCTensor,Union[MPCTensor,int,float,np.ndarray]] : Result of the operation
        """
        # syft absolute
        from syft import parties as mpc_parties

        op = getattr(operator, op_str)

        if isinstance(other, TensorPointer) and self.client != other.client:

            self_mpc = MPCTensor(
                secret=self, shape=self.public_shape, parties=mpc_parties
            )
            other_mpc = MPCTensor(
                secret=other, shape=other.public_shape, parties=mpc_parties
            )

            return op(self_mpc, other_mpc)

        elif isinstance(other, MPCTensor):
            self_mpc = MPCTensor(
                secret=self, shape=self.public_shape, parties=mpc_parties
            )

            return op(self_mpc, other)

        return self._apply_tensor_op(other=other, op_str=op_str)

    def __add__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "add" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "add")

    def __sub__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "sub" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "sub")

    def __mul__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "mul" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "mul")

@bind_protobuf
class Tensor(
    PassthroughTensor,
    AutogradTensorAncestor,
    PhiTensorAncestor,
    FixedPrecisionTensorAncestor,
    # MPCTensorAncestor,
    RecursiveSerde,
):

    __attr_allowlist__ = ["child", "tag_name", "public_shape"]

    PointerClassOverride = TensorPointer

    def __init__(
        self, child: Any, public_shape: Optional[Tuple[int, ...]] = None
    ) -> None:
        """data must be a list of numpy array"""

        if isinstance(child, list):
            child = np.array(child)

        if isinstance(child, th.Tensor):
            child = child.numpy()

        if not isinstance(child, PassthroughTensor) and not isinstance(
            child, np.ndarray
        ):
            raise Exception("Data must be list or nd.array")

        kwargs = {"child": child}
        super().__init__(**kwargs)

        self.tag_name: Optional[str] = None
        self.public_shape = public_shape

    def tag(self, name: str) -> Tensor:
        self.tag_name = name
        return self

    def init_pointer(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
    ) -> Pointer:

        # relative
        from .autodp.single_entity_phi import SingleEntityPhiTensor
        from .autodp.single_entity_phi import TensorWrappedSingleEntityPhiTensorPointer

        if isinstance(self.child, SingleEntityPhiTensor):
            return TensorWrappedSingleEntityPhiTensorPointer(
                entity=self.child.entity,
                client=client,
                id_at_location=id_at_location,
                object_type=object_type,
                tags=tags,
                description=description,
                min_vals=self.child.min_vals,
                max_vals=self.child.max_vals,
                scalar_manager=self.child.scalar_manager,
                public_shape=getattr(self, "public_shape", None),
            )
        else:
            return TensorPointer(
                client=client,
                id_at_location=id_at_location,
                object_type=object_type,
                tags=tags,
                description=description,
                public_shape=getattr(self, "public_shape", None),
            )
<commit_msg>modifiy shape attribute for int, float<commit_after># future
from __future__ import annotations

# stdlib
import operator
from typing import Any
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

# third party
import numpy as np
import torch as th

# relative
from ... import lib
from ...ast.klass import pointerize_args_and_kwargs
from ...core.common.serde.recursive import RecursiveSerde
from ...util import inherit_tags
from ..common.serde.serializable import bind_protobuf
from ..common.uid import UID
from ..node.abstract.node import AbstractNodeClient
from ..node.common.action.run_class_method_action import RunClassMethodAction
from ..pointer.pointer import Pointer
from .ancestors import AutogradTensorAncestor
from .ancestors import PhiTensorAncestor
from .fixed_precision_tensor_ancestor import FixedPrecisionTensorAncestor
from .passthrough import PassthroughTensor  # type: ignore
from .smpc.mpc_tensor import MPCTensor

class TensorPointer(Pointer):

    # Must set these at class init time despite
    # the fact that klass.Class tries to override them (unsuccessfully)
    __name__ = "TensorPointer"
    __module__ = "syft.core.tensor.tensor"

    def __init__(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
        public_shape: Optional[Tuple[int, ...]] = None,
    ):

        super().__init__(
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

        self.public_shape = public_shape

    def share(self, *parties: Tuple[AbstractNodeClient, ...]) -> MPCTensor:

        parties = tuple(list(parties) + [self.client])

        self_mpc = MPCTensor(secret=self, shape=self.public_shape, parties=parties)

        return self_mpc

    def _apply_tensor_op(self, other: Any, op_str: str) -> Any:
        # we want to get the return type which matches the attr_path_and_name
        # so we ask lib_ast for the return type name that matches out
        # attr_path_and_name and then use that to get the actual pointer klass
        # then set the result to that pointer klass

        attr_path_and_name = "syft.core.tensor.tensor.Tensor."
        attr_path_and_name += "__" + op_str + "__"

        result = TensorPointer(client=self.client)

        # QUESTION can the id_at_location be None?
        result_id_at_location = getattr(result, "id_at_location", None)

        if result_id_at_location is not None:
            # first downcast anything primitive which is not already PyPrimitive
            (
                downcast_args,
                downcast_kwargs,
            ) = lib.python.util.downcast_args_and_kwargs(args=[other], kwargs={})

            # then we convert anything which isnt a pointer into a pointer
            pointer_args, pointer_kwargs = pointerize_args_and_kwargs(
                args=downcast_args,
                kwargs=downcast_kwargs,
                client=self.client,
                gc_enabled=False,
            )

            cmd = RunClassMethodAction(
                path=attr_path_and_name,
                _self=self,
                args=pointer_args,
                kwargs=pointer_kwargs,
                id_at_location=result_id_at_location,
                address=self.client.address,
            )
            self.client.send_immediate_msg_without_reply(msg=cmd)

        inherit_tags(
            attr_path_and_name=attr_path_and_name,
            result=result,
            self_obj=self,
            args=[other],
            kwargs={},
        )

        result_public_shape = None

        op = getattr(operator, op_str)

        if isinstance(other, TensorPointer):
            other_shape = other.public_shape
        elif isinstance(other, (int, float)):
            other_shape = (1,)
        elif isinstance(other, np.ndarray):
            other_shape = other.shape
        else:
            raise ValueError(f"Invalid Type for TensorPointer:{type(other)}")

        if self.public_shape is not None and other_shape is not None:
            result_public_shape = (
                op(np.empty(self.public_shape), np.empty(other_shape))
            ).shape

        result.public_shape = result_public_shape

        return result

    @staticmethod
    def _apply_op(
        self: TensorPointer,
        other: Union[TensorPointer, MPCTensor, int, float, np.ndarray],
        op_str: str,
    ) -> Tuple[MPCTensor, Union[MPCTensor, int, float, np.ndarray]]:
        """Performs the operation based on op_str

        Args:
            other (Union[TensorPointer,MPCTensor,int,float,np.ndarray]): second operand.

        Returns:
            Tuple[MPCTensor,Union[MPCTensor,int,float,np.ndarray]] : Result of the operation
        """
        # syft absolute
        from syft import parties as mpc_parties

        op = getattr(operator, op_str)

        if isinstance(other, TensorPointer) and self.client != other.client:

            self_mpc = MPCTensor(
                secret=self, shape=self.public_shape, parties=mpc_parties
            )
            other_mpc = MPCTensor(
                secret=other, shape=other.public_shape, parties=mpc_parties
            )

            return op(self_mpc, other_mpc)

        elif isinstance(other, MPCTensor):
            self_mpc = MPCTensor(
                secret=self, shape=self.public_shape, parties=mpc_parties
            )

            return op(self_mpc, other)

        return self._apply_tensor_op(other=other, op_str=op_str)

    def __add__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "add" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "add")

    def __sub__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "sub" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "sub")

    def __mul__(
        self, other: Union[TensorPointer, MPCTensor, int, float, np.ndarray]
    ) -> Union[TensorPointer, MPCTensor]:
        """Apply the "mul" operation between "self" and "other"

        Args:
            y (Union[TensorPointer,MPCTensor,int,float,np.ndarray]) : second operand.

        Returns:
            Union[TensorPointer,MPCTensor] : Result of the operation.
        """
        return TensorPointer._apply_op(self, other, "mul")

@bind_protobuf
class Tensor(
    PassthroughTensor,
    AutogradTensorAncestor,
    PhiTensorAncestor,
    FixedPrecisionTensorAncestor,
    # MPCTensorAncestor,
    RecursiveSerde,
):

    __attr_allowlist__ = ["child", "tag_name", "public_shape"]

    PointerClassOverride = TensorPointer

    def __init__(
        self, child: Any, public_shape: Optional[Tuple[int, ...]] = None
    ) -> None:
        """data must be a list of numpy array"""

        if isinstance(child, list):
            child = np.array(child)

        if isinstance(child, th.Tensor):
            child = child.numpy()

        if not isinstance(child, PassthroughTensor) and not isinstance(
            child, np.ndarray
        ):
            raise Exception("Data must be list or nd.array")

        kwargs = {"child": child}
        super().__init__(**kwargs)

        self.tag_name: Optional[str] = None
        self.public_shape = public_shape

    def tag(self, name: str) -> Tensor:
        self.tag_name = name
        return self

    def init_pointer(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
    ) -> Pointer:

        # relative
        from .autodp.single_entity_phi import SingleEntityPhiTensor
        from .autodp.single_entity_phi import TensorWrappedSingleEntityPhiTensorPointer

        if isinstance(self.child, SingleEntityPhiTensor):
            return TensorWrappedSingleEntityPhiTensorPointer(
                entity=self.child.entity,
                client=client,
                id_at_location=id_at_location,
                object_type=object_type,
                tags=tags,
                description=description,
                min_vals=self.child.min_vals,
                max_vals=self.child.max_vals,
                scalar_manager=self.child.scalar_manager,
                public_shape=getattr(self, "public_shape", None),
            )
        else:
            return TensorPointer(
                client=client,
                id_at_location=id_at_location,
                object_type=object_type,
                tags=tags,
                description=description,
                public_shape=getattr(self, "public_shape", None),
            )
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""List of known-incompatibly-licensed directories for Android WebView.

This is not used by the webview_licenses tool itself; it is effectively a
"cache" of the output of webview_licenses.GetIncompatibleDirectories() for the
subset of repositories that WebView needs.

We store a copy here because GetIncompatibleDirectories() doesn't work properly
after things have been removed from the tree - it can no longer see the
README.chromium files for previously-removed directories, but they may have
newly added files in them. As long as this list is up to date, we can remove the
things listed first, and then just run the tool afterwards to validate that it
was sufficient. If the tool returns any extra directories then the snapshotting
process will stop and this list must be updated.

"""

KNOWN_ISSUES = {
  'third_party/usb_ids', # crbug.com/161021, doesn't have a LICENSE file
}

KNOWN_INCOMPATIBLE = {
    # Incompatible code in the main chromium repository.
    '.': [
        'base/third_party/xdg_mime',
        'breakpad',
        'chrome/installer/mac/third_party/xz',
        'chrome/test/data',
        'third_party/active_doc',
        'third_party/apple_apsl',
        'third_party/apple_sample_code',
        'third_party/bsdiff',
        'third_party/bspatch',
        'third_party/sudden_motion_sensor',
        'third_party/swiftshader',
        'third_party/talloc',
        'third_party/webdriver',
        'third_party/wtl',
        'tools/telemetry/third_party/websocket-client',
    ],
    # Incompatible code in ICU.
    'third_party/icu': [
        'source/data/brkitr',
    ],
}

for directory in KNOWN_ISSUES:
    KNOWN_INCOMPATIBLE['.'].append(directory)
<commit_msg>Fix syntax in android_webview/tools/known_issues.py<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""List of known-incompatibly-licensed directories for Android WebView.

This is not used by the webview_licenses tool itself; it is effectively a
"cache" of the output of webview_licenses.GetIncompatibleDirectories() for the
subset of repositories that WebView needs.

We store a copy here because GetIncompatibleDirectories() doesn't work properly
after things have been removed from the tree - it can no longer see the
README.chromium files for previously-removed directories, but they may have
newly added files in them. As long as this list is up to date, we can remove the
things listed first, and then just run the tool afterwards to validate that it
was sufficient. If the tool returns any extra directories then the snapshotting
process will stop and this list must be updated.

"""

KNOWN_ISSUES = set([
    'third_party/usb_ids', # crbug.com/161021, doesn't have a LICENSE file
])

KNOWN_INCOMPATIBLE = {
    # Incompatible code in the main chromium repository.
    '.': [
        'base/third_party/xdg_mime',
        'breakpad',
        'chrome/installer/mac/third_party/xz',
        'chrome/test/data',
        'third_party/active_doc',
        'third_party/apple_apsl',
        'third_party/apple_sample_code',
        'third_party/bsdiff',
        'third_party/bspatch',
        'third_party/sudden_motion_sensor',
        'third_party/swiftshader',
        'third_party/talloc',
        'third_party/webdriver',
        'third_party/wtl',
        'tools/telemetry/third_party/websocket-client',
    ],
    # Incompatible code in ICU.
    'third_party/icu': [
        'source/data/brkitr',
    ],
}

for directory in KNOWN_ISSUES:
    KNOWN_INCOMPATIBLE['.'].append(directory)
<|endoftext|>


<commit_before>import os
import shutil
import sys
from oct.multimechanize.utilities.newproject import CONFIG_NAME, SCRIPT_NAME, \
    SCRIPTS_DIR

TEMPLATE_DIR = 'templates'

BASE_DIR = os.path.abspath(os.path.dirname(__file__))

HEAD_CONTENT = """
<!DOCTYPE html>
<html>

<head>
    <title> OCT | Results </title>
    <link rel="stylesheet" type="text/css" href="css/style.css" />
</head>

<body id="main">
"""

FOOTER_CONTENT = """
</body>
</html>
"""

SCRIPT_CONTENT = """
from oct.core.generic import GenericTransaction
import random
import time
import os

CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../')

class Transaction(GenericTransaction):
    def __init__(self):
        GenericTransaction.__init__(self, True, CONFIG_PATH)

    def run(self):
        r = random.uniform(1, 2)
        time.sleep(r)
        self.custom_timers['Example_Timer'] = r

if __name__ == '__main__':
    trans = Transaction()
    trans.run()
    print trans.custom_timers
"""

CONFIG_CONTENT = """
[global]
run_time = 30
rampup = 0
results_ts_interval = 10
progress_bar = on
console_logging = off
xml_report = off
base_url = http://localhost
default_sleep_time = 2

[user_group-1]
threads = 3
script = %s

[user_group-2]
threads = 3
script = %s

""" % (SCRIPT_NAME, SCRIPT_NAME)

def create_project(
        project_name,
        config_name=CONFIG_NAME,
        script_name=SCRIPT_NAME,
        scripts_dir=SCRIPTS_DIR,
        config_content=CONFIG_CONTENT,
        script_content=SCRIPT_CONTENT,
        template_dir=TEMPLATE_DIR,
        head_content=HEAD_CONTENT,
        footer_content=FOOTER_CONTENT):

    if os.path.exists(project_name):
        sys.stderr.write('\nERROR: project already exists: %s\n\n' % project_name)
        sys.exit(1)
    try:
        os.makedirs(project_name)
        os.makedirs(os.path.join(project_name, scripts_dir))
        os.makedirs(os.path.join(project_name, template_dir))
        os.makedirs(os.path.join(project_name, template_dir, 'css'))
        os.makedirs(os.path.join(project_name, template_dir, 'scripts'))
        os.makedirs(os.path.join(project_name, template_dir, 'img'))
        shutil.copy(os.path.join(BASE_DIR, 'templates', 'css', 'style.css'),
                    os.path.join(project_name, template_dir, 'css'))
    except OSError as e:
        sys.stderr.write('\nERROR: can not create directory for %r\n\n' % project_name)
        sys.exit(1)
    with open(os.path.join(project_name, config_name), 'w') as f:
        f.write(config_content)
    with open(os.path.join(project_name, scripts_dir, script_name), 'w') as f:
        f.write(script_content)
    with open(os.path.join(project_name, template_dir, 'head.html'), 'w') as f:
        f.write(head_content)
    with open(os.path.join(project_name, template_dir, 'footer.html'), 'w') as f:
        f.write(footer_content)

def main():
    try:
        project_name = sys.argv[1]
    except IndexError:
        sys.stderr.write('\nERROR: no project specified\n\n')
        sys.stderr.write('Usage: oct-newproject <project name>\n\n')
        sys.exit(1)

    create_project(project_name)

if __name__ == '__main__':
    main()

<commit_msg>update newproject command to add a default value to statics_enabled option<commit_after>import os
import shutil
import sys
from oct.multimechanize.utilities.newproject import CONFIG_NAME, SCRIPT_NAME, \
    SCRIPTS_DIR

TEMPLATE_DIR = 'templates'

BASE_DIR = os.path.abspath(os.path.dirname(__file__))

HEAD_CONTENT = """
<!DOCTYPE html>
<html>

<head>
    <title> OCT | Results </title>
    <link rel="stylesheet" type="text/css" href="css/style.css" />
</head>

<body id="main">
"""

FOOTER_CONTENT = """
</body>
</html>
"""

SCRIPT_CONTENT = """
from oct.core.generic import GenericTransaction
import random
import time
import os

CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../')

class Transaction(GenericTransaction):
    def __init__(self):
        GenericTransaction.__init__(self, True, CONFIG_PATH)

    def run(self):
        r = random.uniform(1, 2)
        time.sleep(r)
        self.custom_timers['Example_Timer'] = r

if __name__ == '__main__':
    trans = Transaction()
    trans.run()
    print trans.custom_timers
"""

CONFIG_CONTENT = """
[global]
run_time = 30
rampup = 0
results_ts_interval = 10
progress_bar = on
console_logging = off
xml_report = off
base_url = http://localhost
default_sleep_time = 2
statics_enabled = 1

[user_group-1]
threads = 3
script = %s

[user_group-2]
threads = 3
script = %s

""" % (SCRIPT_NAME, SCRIPT_NAME)

def create_project(
        project_name,
        config_name=CONFIG_NAME,
        script_name=SCRIPT_NAME,
        scripts_dir=SCRIPTS_DIR,
        config_content=CONFIG_CONTENT,
        script_content=SCRIPT_CONTENT,
        template_dir=TEMPLATE_DIR,
        head_content=HEAD_CONTENT,
        footer_content=FOOTER_CONTENT):

    if os.path.exists(project_name):
        sys.stderr.write('\nERROR: project already exists: %s\n\n' % project_name)
        sys.exit(1)
    try:
        os.makedirs(project_name)
        os.makedirs(os.path.join(project_name, scripts_dir))
        os.makedirs(os.path.join(project_name, template_dir))
        os.makedirs(os.path.join(project_name, template_dir, 'css'))
        os.makedirs(os.path.join(project_name, template_dir, 'scripts'))
        os.makedirs(os.path.join(project_name, template_dir, 'img'))
        shutil.copy(os.path.join(BASE_DIR, 'templates', 'css', 'style.css'),
                    os.path.join(project_name, template_dir, 'css'))
    except OSError as e:
        sys.stderr.write('\nERROR: can not create directory for %r\n\n' % project_name)
        sys.exit(1)
    with open(os.path.join(project_name, config_name), 'w') as f:
        f.write(config_content)
    with open(os.path.join(project_name, scripts_dir, script_name), 'w') as f:
        f.write(script_content)
    with open(os.path.join(project_name, template_dir, 'head.html'), 'w') as f:
        f.write(head_content)
    with open(os.path.join(project_name, template_dir, 'footer.html'), 'w') as f:
        f.write(footer_content)

def main():
    try:
        project_name = sys.argv[1]
    except IndexError:
        sys.stderr.write('\nERROR: no project specified\n\n')
        sys.stderr.write('Usage: oct-newproject <project name>\n\n')
        sys.exit(1)

    create_project(project_name)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import os
import sys
from django.contrib.messages import constants as messages

# Directory that contains this file.
PROJECT_SETTINGS_DIRECTORY = os.path.dirname(globals()['__file__'])
# Root directory. Contains manage.py
PROJECT_ROOT_DIRECTORY = os.path.join(PROJECT_SETTINGS_DIRECTORY, '..', '..')
#PROJECT_ROOT_DIRECTORY = os.path.dirname(os.path.dirname(__file__))

TEST_RUNNER = "django_nose.NoseTestSuiteRunner"
NOSE_ARGS = ['--with-coverage', '--cover-package=apps']

DEBUG = False
TEMPLATE_DEBUG = DEBUG

TEST_RUNNER = "django_nose.NoseTestSuiteRunner"

ADMINS = (
    ('dotKom', 'dotkom@online.ntnu.no'),
)
MANAGERS = ADMINS

# Email settings
DEFAULT_FROM_EMAIL = 'online@online.ntnu.no'
EMAIL_ARRKOM = 'arrkom@online.ntnu.no'
EMAIL_BEDKOM = 'bedkom@online.ntnu.no'
EMAIL_DOTKOM = 'dotkom@online.ntnu.no'
EMAIL_FAGKOM = 'fagkom@online.ntnu.no'
EMAIL_PROKOM = 'prokom@online.ntnu.no'
EMAIL_TRIKOM = 'trikom@online.ntnu.no'

# We will receive errors and other django messages from this email
SERVER_EMAIL = 'onlineweb4-error@online.ntnu.no'

TIME_ZONE = 'Europe/Oslo'
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'nb'
LANGUAGES = (
                ('nb', 'Norwegian'),
                ('en_US', 'English'),
            )

SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = False
DATETIME_FORMAT = 'N j, Y, H:i'
SECRET_KEY = 'q#wy0df(7&$ucfrxa1j72%do7ko*-6(g!8f$tc2$3x@3cq5@6c'

AUTH_USER_MODEL = 'authentication.OnlineUser'
LOGIN_URL = '/auth/login/'

MEDIA_ROOT = os.path.join(PROJECT_ROOT_DIRECTORY, 'uploaded_media') # Override this in local.py in prod.
MEDIA_URL = '/media/'

STATIC_ROOT = os.path.join(PROJECT_ROOT_DIRECTORY, 'static')
STATIC_URL = '/static/'

#Url of default profile picture
DEFAULT_PROFILE_PICTURE_URL = os.path.join(STATIC_URL, "img", "profile_default.png")

# Additional locations of static files
STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT_DIRECTORY, 'files/static'),
)

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
    'compressor.finders.CompressorFinder',
)

COMPRESS_OUTPUT_DIR = 'cache'
COMPRESS_PRECOMPILERS = (
    ('text/less', 'lessc {infile} {outfile}'),
)

COMPRESS_CSS_FILTERS = [
    'compressor.filters.css_default.CssAbsoluteFilter',
    'compressor-filters.cssmin.CSSMinFilter',
]
COMPRESS_JS_FILTERS = [
    'compressor.filters.jsmin.JSMinFIlter',
]

COMPRESS_PRECOMPILERS = (
    ('text/less', 'lessc -x {infile} {outfile}'),
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'middleware.http.Http403Middleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'onlineweb4.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'onlineweb4.wsgi.application'

TEMPLATE_DIRS = (
    os.path.join(PROJECT_ROOT_DIRECTORY, 'templates/')
)

# Pizzasystem settings
PIZZA_GROUP = 'dotkom'
PIZZA_ADMIN_GROUP = 'pizzaadmin'

# Grappelli settings
GRAPPELLI_ADMIN_TITLE = 'Onlineweb'

INSTALLED_APPS = (
    # Third party dependencies
    'django_nose',
    'south',
    'grappelli',
    'filebrowser',
    'chunks',
    'crispy_forms',
    'django_extensions',
    'django_dynamic_fixture',
    'captcha',
    'compressor',
    'pizzasystem',

    # Django apps
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.markup',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    # Onlineweb 4 apps
    'apps.article',
    'apps.authentication',
    'apps.autoconfig',
    'apps.careeropportunity',
    'apps.companyprofile',
    'apps.events',
    'apps.marks',
    'apps.offline',
    'apps.feedback',
    'apps.mommy',
    'apps.profiles',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
        },
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
        'apscheduler.scheduler': {
            'handlers': ['console'],
            'level': 'INFO',
        },
    }
}

# crispy forms settings
CRISPY_TEMPLATE_PACK = 'bootstrap3'

# bootstrap messages classes
MESSAGE_TAGS = {messages.DEBUG: 'alert-debug',
                messages.INFO: 'alert-info',
                messages.SUCCESS: 'alert-success',
                messages.WARNING: 'alert-warning',
                messages.ERROR: 'alert-error'}

# Not really sure what this does.
# Has something to do with django-dynamic-fixture bumped from 1.6.4 to 1.6.5 in order to run a syncdb with mysql/postgres (OptimusCrime)
IMPORT_DDF_MODELS = False

# Remember to keep 'local' last, so it can override any setting.
for settings_module in ['filebrowser', 'local']:  # local last
    if not os.path.exists(os.path.join(PROJECT_SETTINGS_DIRECTORY,
            settings_module + ".py")):
        sys.stderr.write("Could not find settings module '%s'.\n" %
                settings_module)
        if settings_module == 'local':
            sys.stderr.write("You need to copy the settings file "
                             "'onlineweb4/settings/example-local.py' to "
                             "'onlineweb4/settings/local.py'.\n")
        sys.exit(1)
    try:
        exec('from %s import *' % settings_module)
    except ImportError, e:
        print "Could not import settings for '%s' : %s" % (settings_module,
                str(e))
<commit_msg>Added LOGIN_URL for redirection<commit_after>import os
import sys
from django.contrib.messages import constants as messages

# Directory that contains this file.
PROJECT_SETTINGS_DIRECTORY = os.path.dirname(globals()['__file__'])
# Root directory. Contains manage.py
PROJECT_ROOT_DIRECTORY = os.path.join(PROJECT_SETTINGS_DIRECTORY, '..', '..')
#PROJECT_ROOT_DIRECTORY = os.path.dirname(os.path.dirname(__file__))

TEST_RUNNER = "django_nose.NoseTestSuiteRunner"
NOSE_ARGS = ['--with-coverage', '--cover-package=apps']

DEBUG = False
TEMPLATE_DEBUG = DEBUG

TEST_RUNNER = "django_nose.NoseTestSuiteRunner"

ADMINS = (
    ('dotKom', 'dotkom@online.ntnu.no'),
)
MANAGERS = ADMINS

# Email settings
DEFAULT_FROM_EMAIL = 'online@online.ntnu.no'
EMAIL_ARRKOM = 'arrkom@online.ntnu.no'
EMAIL_BEDKOM = 'bedkom@online.ntnu.no'
EMAIL_DOTKOM = 'dotkom@online.ntnu.no'
EMAIL_FAGKOM = 'fagkom@online.ntnu.no'
EMAIL_PROKOM = 'prokom@online.ntnu.no'
EMAIL_TRIKOM = 'trikom@online.ntnu.no'

# We will receive errors and other django messages from this email
SERVER_EMAIL = 'onlineweb4-error@online.ntnu.no'

TIME_ZONE = 'Europe/Oslo'
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'nb'
LANGUAGES = (
                ('nb', 'Norwegian'),
                ('en_US', 'English'),
            )

SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = False
DATETIME_FORMAT = 'N j, Y, H:i'
SECRET_KEY = 'q#wy0df(7&$ucfrxa1j72%do7ko*-6(g!8f$tc2$3x@3cq5@6c'

AUTH_USER_MODEL = 'authentication.OnlineUser'
LOGIN_URL = '/auth/login/'

LOGIN_URL = '/auth/login/'

MEDIA_ROOT = os.path.join(PROJECT_ROOT_DIRECTORY, 'uploaded_media') # Override this in local.py in prod.
MEDIA_URL = '/media/'

STATIC_ROOT = os.path.join(PROJECT_ROOT_DIRECTORY, 'static')
STATIC_URL = '/static/'

#Url of default profile picture
DEFAULT_PROFILE_PICTURE_URL = os.path.join(STATIC_URL, "img", "profile_default.png")

# Additional locations of static files
STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT_DIRECTORY, 'files/static'),
)

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
    'compressor.finders.CompressorFinder',
)

COMPRESS_OUTPUT_DIR = 'cache'
COMPRESS_PRECOMPILERS = (
    ('text/less', 'lessc {infile} {outfile}'),
)

COMPRESS_CSS_FILTERS = [
    'compressor.filters.css_default.CssAbsoluteFilter',
    'compressor-filters.cssmin.CSSMinFilter',
]
COMPRESS_JS_FILTERS = [
    'compressor.filters.jsmin.JSMinFIlter',
]

COMPRESS_PRECOMPILERS = (
    ('text/less', 'lessc -x {infile} {outfile}'),
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'middleware.http.Http403Middleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'onlineweb4.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'onlineweb4.wsgi.application'

TEMPLATE_DIRS = (
    os.path.join(PROJECT_ROOT_DIRECTORY, 'templates/')
)

# Pizzasystem settings
PIZZA_GROUP = 'dotkom'
PIZZA_ADMIN_GROUP = 'pizzaadmin'

# Grappelli settings
GRAPPELLI_ADMIN_TITLE = 'Onlineweb'

INSTALLED_APPS = (
    # Third party dependencies
    'django_nose',
    'south',
    'grappelli',
    'filebrowser',
    'chunks',
    'crispy_forms',
    'django_extensions',
    'django_dynamic_fixture',
    'captcha',
    'compressor',
    'pizzasystem',

    # Django apps
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.markup',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    # Onlineweb 4 apps
    'apps.article',
    'apps.authentication',
    'apps.autoconfig',
    'apps.careeropportunity',
    'apps.companyprofile',
    'apps.events',
    'apps.marks',
    'apps.offline',
    'apps.feedback',
    'apps.mommy',
    'apps.profiles',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
        },
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
        'apscheduler.scheduler': {
            'handlers': ['console'],
            'level': 'INFO',
        },
    }
}

# crispy forms settings
CRISPY_TEMPLATE_PACK = 'bootstrap3'

# bootstrap messages classes
MESSAGE_TAGS = {messages.DEBUG: 'alert-debug',
                messages.INFO: 'alert-info',
                messages.SUCCESS: 'alert-success',
                messages.WARNING: 'alert-warning',
                messages.ERROR: 'alert-error'}

# Not really sure what this does.
# Has something to do with django-dynamic-fixture bumped from 1.6.4 to 1.6.5 in order to run a syncdb with mysql/postgres (OptimusCrime)
IMPORT_DDF_MODELS = False

# Remember to keep 'local' last, so it can override any setting.
for settings_module in ['filebrowser', 'local']:  # local last
    if not os.path.exists(os.path.join(PROJECT_SETTINGS_DIRECTORY,
            settings_module + ".py")):
        sys.stderr.write("Could not find settings module '%s'.\n" %
                settings_module)
        if settings_module == 'local':
            sys.stderr.write("You need to copy the settings file "
                             "'onlineweb4/settings/example-local.py' to "
                             "'onlineweb4/settings/local.py'.\n")
        sys.exit(1)
    try:
        exec('from %s import *' % settings_module)
    except ImportError, e:
        print "Could not import settings for '%s' : %s" % (settings_module,
                str(e))
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Routes for end to end testing, can only be called when in test mode.

'''
__author__ = 'M@Campbell'

from flask import jsonify, current_app, request
from ooiservices.app.main import api
from celery.task.control import discard_all
import urllib
import subprocess

@api.route('/list_routes', methods=['GET'])
def list_routes():
    """
    Create a list of (url, endpoint) tuples from app.url_map
    Used for performance testing
    :return:
    """
    routes = []
    for rule in current_app.url_map.iter_rules():
        if 'GET' in rule.methods:
            url = urllib.unquote("{}".format(rule))
            routes.append((url, rule.endpoint))

    if not routes:
        return '{}', 204

    return jsonify({'routes': routes})

@api.route('/cache_keys', methods=['GET'])
@api.route('/cache_keys/<string:key>', methods=['DELETE'])
def cache_list(key=None):
    '''
    @method GET:
        Returns this list of flask cache keys and their
        time to live (TTL)
        :return: JSON

    @method DELETE:
        Send a delete request passing in the name of the
        cache key that needs to be deleted.
        :return: JSON
    '''
    if request.method == 'GET':
        # setup the container for the list of current cache items
        flask_cache = []

        # issue the command to get the list from redis
        pipe_output = subprocess.Popen(['redis-cli', 'scan', '0'],
                                       stdout=subprocess.PIPE)
        output, err = pipe_output.communicate()

        # the output is a string, so lets load it into the array
        flask_cache = output.split('\n')

        # we don't want to provide system keys, remove non "flask_cache"
        temp_list = []
        for cache_key in flask_cache:
            if 'flask_cache' in cache_key:
                temp_list.append({'key': cache_key,
                                  'name': cache_key.split('_')[2]
                                  })

        # assign the list to the main bin
        flask_cache = temp_list

        # lets get the TTL of each of the keys
        for cache_key in flask_cache:
            pipe_output = subprocess.Popen(['redis-cli', 'TTL',
                                            cache_key['key']],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()
            cache_key['TTL'] = output.split('\n')[0]

        # clear out this for garbage collection of flask_cache ref
        temp_list = None

        return jsonify({'results': flask_cache})

    elif request.method == 'DELETE':
        try:
            # grab the key, lets give it a more readable reference
            redis_key = key

            # perform the delete operation on the target host's redis server
            pipe_output = subprocess.Popen(['redis-cli', 'del', redis_key],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()

            # we'll discard all previous celery jobs, so new ones can be queued up.
            # Note, this does NOT issue a cache reload.  It simply removes any
            # backloged tasks.  Keep it tight, like a tiger.
            if output == 1:
                discard_all()

            return jsonify({'results': int(output)}), 200

        except Exception as e:
            return jsonify({'error': 'Exception in cache delete. %s' % e}), 500
<commit_msg>Changes scan to keys to match redis-cli app.<commit_after>#!/usr/bin/env python
'''
Routes for end to end testing, can only be called when in test mode.

'''
__author__ = 'M@Campbell'

from flask import jsonify, current_app, request
from ooiservices.app.main import api
from celery.task.control import discard_all
import urllib
import subprocess

@api.route('/list_routes', methods=['GET'])
def list_routes():
    """
    Create a list of (url, endpoint) tuples from app.url_map
    Used for performance testing
    :return:
    """
    routes = []
    for rule in current_app.url_map.iter_rules():
        if 'GET' in rule.methods:
            url = urllib.unquote("{}".format(rule))
            routes.append((url, rule.endpoint))

    if not routes:
        return '{}', 204

    return jsonify({'routes': routes})

@api.route('/cache_keys', methods=['GET'])
@api.route('/cache_keys/<string:key>', methods=['DELETE'])
def cache_list(key=None):
    '''
    @method GET:
        Returns this list of flask cache keys and their
        time to live (TTL)
        :return: JSON

    @method DELETE:
        Send a delete request passing in the name of the
        cache key that needs to be deleted.
        :return: JSON
    '''
    if request.method == 'GET':
        # setup the container for the list of current cache items
        flask_cache = []

        # issue the command to get the list from redis
        pipe_output = subprocess.Popen(['redis-cli', 'keys', '*'],
                                       stdout=subprocess.PIPE)
        output, err = pipe_output.communicate()

        # the output is a string, so lets load it into the array
        flask_cache = output.split('\n')

        # we don't want to provide system keys, remove non "flask_cache"
        temp_list = []
        for cache_key in flask_cache:
            if 'flask_cache' in cache_key:
                temp_list.append({'key': cache_key,
                                  'name': cache_key.split('_')[2]
                                  })

        # assign the list to the main bin
        flask_cache = temp_list

        # lets get the TTL of each of the keys
        for cache_key in flask_cache:
            pipe_output = subprocess.Popen(['redis-cli', 'TTL',
                                            cache_key['key']],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()
            cache_key['TTL'] = output.split('\n')[0]

        # clear out this for garbage collection of flask_cache ref
        temp_list = None

        return jsonify({'results': flask_cache})

    elif request.method == 'DELETE':
        try:
            # grab the key, lets give it a more readable reference
            redis_key = key

            # perform the delete operation on the target host's redis server
            pipe_output = subprocess.Popen(['redis-cli', 'del', redis_key],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()

            # we'll discard all previous celery jobs, so new ones can be queued up.
            # Note, this does NOT issue a cache reload.  It simply removes any
            # backloged tasks.  Keep it tight, like a tiger.
            if output == 1:
                discard_all()

            return jsonify({'results': int(output)}), 200

        except Exception as e:
            return jsonify({'error': 'Exception in cache delete. %s' % e}), 500
<|endoftext|>