<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import ftplib
import logging
import os
import requests
import shutil
import sys
import tempfile

import stoneridge

LINUX_SUBDIRS = ('try-linux', 'try-linux64')
MAC_SUBDIRS = ('try-macosx64',) # There is only one OS X build
WINDOWS_SUBDIRS = ('try-win32',) # win64 is unsupported, so ignore it for now

class StoneRidgeCloner(object):
    """This runs on the central stone ridge server, and downloads releases from
    ftp.m.o to a local directory that is served up to the clients by a plain ol'
    web server. Those clients use stoneridge_downloader.py to get the files they
    need from the central server.
    """
    def __init__(self, nightly, srid, operating_systems, netconfigs,
            ldap, sha, attempt):
        self.host = stoneridge.get_config('cloner', 'host')
        self.nightly = nightly
        self.outroot = stoneridge.get_config('cloner', 'output')
        self.srid = srid
        self.outdir = os.path.join(self.outroot, srid)
        self.keep = stoneridge.get_config_int('cloner', 'keep', default=50)
        self.max_attempts = stoneridge.get_config_int('cloner', 'attempts')
        self.operating_systems = operating_systems
        self.netconfigs = netconfigs
        self.ldap = ldap
        self.sha = sha
        self.attempt = attempt

        if not os.path.exists(self.outroot):
            os.mkdir(self.outroot)

        root = stoneridge.get_config('cloner', 'root')
        if nightly:
            self.path = '/'.join([root, 'nightly', 'latest-mozilla-central'])
        else:
            self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)])

        logging.debug('host: %s' % (self.host,))
        logging.debug('path: %s' % (self.path,))
        logging.debug('nightly: %s' % (self.nightly,))
        logging.debug('srid: %s' % (self.srid,))
        logging.debug('output root: %s' % (self.outroot,))
        logging.debug('output directory: %s' % (self.outdir,))
        logging.debug('keep history: %s' % (self.keep,))
        logging.debug('max attempts: %s' % (self.max_attempts,))
        logging.debug('operating systems: %s' % (self.operating_systems,))
        logging.debug('netconfigs: %s' % (self.netconfigs,))
        logging.debug('ldap: %s' % (self.ldap,))
        logging.debug('sha: %s' % (self.sha,))
        logging.debug('attempt: %s' % (self.attempt,))

        self.prefix = ''

    def _gather_filelist(self, path):
        """Get the list of files available on our FTP server

        Returns: list of filenames relative to the path on the server
        """
        logging.debug('gathering files from ftp server')

        try:
            ftp = ftplib.FTP(self.host)
            ftp.login()
            ftp.cwd(path)
            files = ftp.nlst()
            ftp.quit()
        except:
            # We blanket-catch exceptions here, because we want the error
            # handling in the top level to take precedence for ANY problem that
            # happens while listing the directory. Logging helps us track down
            # unexpected errors that may occur.
            logging.exception('Unable to list files in %s' % (path,))
            return []

        logging.debug('files in %s: %s' % (path, files))
        return files

    def _build_dl_url(self, try_subdir, fname):
        """Create a download (https) URL for a particular file

        Returns: a URL string
        """
        logging.debug('creating download url for %s' % (fname,))
        remotefile = self.path
        if not self.nightly:
            remotefile = '/'.join([remotefile, try_subdir])
        remotefile = '/'.join([remotefile, fname])
        logging.debug('remote filename: %s' % (remotefile,))
        url = 'https://%s%s' % (self.host, remotefile)
        logging.debug('url: %s' % (url,))
        return url

    def _get_prefix(self, files):
        """Get the filename prefix that is common to all the files we'll need to
        download

        Returns: <prefix (string)>
        """
        logging.debug('getting filename prefix')
        prefixfile = [f for f in files if f.endswith('.checksums.asc')][-1]
        prefix = prefixfile.replace('.checksums.asc', '')
        prefix = prefix.rsplit('.', 1)[0] # Strip off the platform information
        logging.debug('filename prefix: %s' % (prefix,))
        return prefix

    def _ensure_outdir(self, platform):
        """Ensure the output directory for a platform exists
        """
        logging.debug('ensuring output directory for %s exists' % (platform,))
        if not os.path.exists(self.outdir):
            logging.debug('creating outdir %s' % (self.outdir,))
            os.mkdir(self.outdir)
        platdir = os.path.join(self.outdir, platform)
        logging.debug('platform directory: %s' % (platdir,))
        if not os.path.exists(platdir):
            logging.debug('creating platform directory %s' % (platdir,))
            os.mkdir(platdir)

    def _dl_to_file(self, url, outfile):
        """Download the file at <url> and save it to the file
        at <outfile>
        """
        logging.debug('downloading %s => %s' % (url, outfile))
        resp = requests.get(url, timeout=30000)
        with file(outfile, 'wb') as f:
            logging.debug('writing file contents')
            f.write(resp.content)

    def _dl_test_zip(self, try_subdir, archid, outdir):
        """Download the test zip for a particular architecture id (<archid>) and
        save it at <outdir>/tests.zip
        """
        logging.debug('downloading test zip for %s to %s' % (archid, outdir))
        srcfile = '%s.%s.tests.zip' % (self.prefix, archid)
        logging.debug('zip source filename: %s' % (srcfile,))
        url = self._build_dl_url(try_subdir, srcfile)
        outfile = os.path.join(self.outdir, outdir, 'tests.zip')
        logging.debug('zip dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

    def _clone_mac(self):
        """Clone the dmg and tests zip for the mac build
        """
        logging.debug('cloning mac build')
        self._ensure_outdir('mac')

        logging.debug('downloading firefox dmg')
        dmg = '%s.mac.dmg' % (self.prefix,)
        logging.debug('dmg source filename: %s' % (dmg,))
        url = self._build_dl_url(MAC_SUBDIRS[0], dmg)
        outfile = os.path.join(self.outdir, 'mac', 'firefox.dmg')
        logging.debug('dmg dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

        self._dl_test_zip(MAC_SUBDIRS[0], 'mac', 'mac')

    def _clone_linux(self):
        """Clone the .tar.bz2 and tests zip for both 32-bit and 64-bit linux
        builds
        """
        logging.debug('cloning linux builds')
        archids = ('i686', 'x86_64')
        outdirs = ('linux32', 'linux64')
        for archid, outdir, subdir in zip(archids, outdirs, LINUX_SUBDIRS):
            logging.debug('architecture: %s' % (archid,))
            logging.debug('outdir: %s' % (outdir,))
            self._ensure_outdir(outdir)

            logging.debug('downloading firefox tarball')
            srcfile = '%s.linux-%s.tar.bz2' % (self.prefix, archid)
            logging.debug('tarball source filename: %s' % (srcfile,))
            url = self._build_dl_url(subdir, srcfile)
            outfile = os.path.join(self.outdir, outdir, 'firefox.tar.bz2')
            logging.debug('tarball dest filename: %s' % (outfile,))
            self._dl_to_file(url, outfile)

            self._dl_test_zip(subdir, 'linux-%s' % (archid,), outdir)

    def _clone_win(self):
        """Clone the firefox zip and tests zip for both 32-bit and 64-bit
        windows builds
        """
        logging.debug('cloning windows build')
        self._ensure_outdir('win32')

        logging.debug('downloading firefox zip')
        srcfile = '%s.win32.zip' % (self.prefix,)
        logging.debug('zip source filename: %s' % (srcfile,))
        url = self._build_dl_url(WINDOWS_SUBDIRS[0], srcfile)
        outfile = os.path.join(self.outdir, 'win32', 'firefox.zip')
        logging.debug('zip dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

        self._dl_test_zip(WINDOWS_SUBDIRS[0], 'win32', 'win32')

    def _cleanup_old_directories(self):
        """We only keep around so many directories of historical firefoxen. This
        gets rid of ones we don't care about any more
        """
        logging.debug('cleaning up old directories')
        with stoneridge.cwd(self.outroot):
            listing = os.listdir('.')
            logging.debug('candidate files: %s' %  (listing,))

            # We want to make sure that we're not looking at anything that's not
            # a directory that may have somehow gotten into our directory. We
            # also need to ignore dotfiles.
            directories = [l for l in listing
                           if os.path.isdir(l) and not l.startswith('.')]
            logging.debug('directories: %s' % (directories,))

            # Find out when the directories were last modified, and sort the
            # list by that, so we can delete the oldest ones.
            times = [(d, os.stat(d).st_mtime) for d in directories]
            times.sort(key=lambda x: x[1])

            # Now we can figure out which directories to delete!
            delete_us = times[:-self.keep]
            logging.debug('directories to delete: %s' % (delete_us,))

            for d in delete_us:
                logging.debug('removing %s' % (d,))
                shutil.rmtree(d)

    def defer(self):
        args = ['srdeferrer.py',
                '--srid', self.srid,
                '--config', stoneridge.get_config_file(),
                '--log', '/dev/null',
                '--pidfile', tempfile.mktemp(),
                '--attempt', self.attempt + 1]

        if self.nightly:
            args.append('--nightly')
        else:
            args.extend(['--ldap', self.ldap])
            args.extend(['--sha', self.sha])
            for ops in self.operating_systems:
                args.append('--%s' % (ops,))
            for nc in self.netconfigs:
                args.append('--%s' % (nc,))

        stoneridge.run_process(*args)

        sys.exit(0)

    def exit_and_maybe_defer(self, deferred_message):
        next_attempt = self.attempt + 1
        if next_attempt > self.max_attempts:
            logging.error('Unable to get build results for %s after %s '
                    'attempts. Cancelling run.' %
                    (self.srid, self.max_attempts))
        else:
            self.defer()
            logging.debug(deferred_message)
        sys.exit(1)

    def run(self):
        files = self._gather_filelist(self.path)
        if not self.nightly:
            # For some ungodly reason, try builds have a different directory
            # structure than nightly builds, so we have to handle them
            # differently. Instead of all output being at the same level,
            # they are separated out by platform for try builds. Le sigh.
            subdirs = []
            dist_files = None
            if 'linux' in self.operating_systems:
                subdirs.extend(LINUX_SUBDIRS)
            if 'mac' in self.operating_systems:
                subdirs.extend(MAC_SUBDIRS)
            if 'windows' in self.operating_systems:
                subdirs.extend(WINDOWS_SUBDIRS)

            # Be reasonably sure the try run is complete, such that everything
            # is ready for us to download.
            for d in subdirs:
                if d not in files:
                    self.exit_and_maybe_defer(
                            'Run %s not available: retry later' % (d,))

            dist_path = '/'.join([self.path, subdirs[0]])
            dist_files = self._gather_filelist(dist_path)

            if not dist_files:
                # We didn't get any files listed, but we should have. Just drop
                # this run on the floor
                logging.error('No files found! Dropping srid %s' % (self.srid,))
                sys.exit(1)

            files = dist_files

        if not files:
            self.exit_and_maybe_defer(
                    'No files found for %s: retry later' % (self.srid,))

        self.prefix = self._get_prefix(files)

        # Make sure our output directory exists
        if not os.path.exists(self.outdir):
            logging.debug('creating output directory')
            os.mkdir(self.outdir)

        # Now download all the builds and test zipfiles
        if self.nightly or 'mac' in self.operating_systems:
            self._clone_mac()
        if self.nightly or 'linux' in self.operating_systems:
            self._clone_linux()
        if self.nightly or 'windows' in self.operating_systems:
            self._clone_win()

        self._cleanup_old_directories()

@stoneridge.main
def main():
    parser = stoneridge.ArgumentParser()
    parser.add_argument('--nightly', dest='nightly', action='store_true',
            default=False)
    parser.add_argument('--srid', dest='srid', required=True)
    for ops in stoneridge.OPERATING_SYSTEMS:
        parser.add_argument('--%s' % (ops,), dest='operating_systems',
                action='append_const', const=ops, default=[])
    for nc in stoneridge.NETCONFIGS:
        parser.add_argument('--%s' % (nc,), dest='netconfigs',
                action='append_const', const=nc, default=[])
    parser.add_argument('--attempt', dest='attempt', required=True, type=int)
    parser.add_argument('--ldap', dest='ldap', default='')
    parser.add_argument('--sha', dest='sha', default='')
    args = parser.parse_args()

    cloner = StoneRidgeCloner(args.nightly, args.srid, args.operating_systems,
            args.netconfigs, args.ldap, args.sha, args.attempt)
    cloner.run()
<commit_msg>Fix cloner in deferred case<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import ftplib
import logging
import os
import requests
import shutil
import sys
import tempfile

import stoneridge

LINUX_SUBDIRS = ('try-linux', 'try-linux64')
MAC_SUBDIRS = ('try-macosx64',) # There is only one OS X build
WINDOWS_SUBDIRS = ('try-win32',) # win64 is unsupported, so ignore it for now

class StoneRidgeCloner(object):
    """This runs on the central stone ridge server, and downloads releases from
    ftp.m.o to a local directory that is served up to the clients by a plain ol'
    web server. Those clients use stoneridge_downloader.py to get the files they
    need from the central server.
    """
    def __init__(self, nightly, srid, operating_systems, netconfigs,
            ldap, sha, attempt):
        self.host = stoneridge.get_config('cloner', 'host')
        self.nightly = nightly
        self.outroot = stoneridge.get_config('cloner', 'output')
        self.srid = srid
        self.outdir = os.path.join(self.outroot, srid)
        self.keep = stoneridge.get_config_int('cloner', 'keep', default=50)
        self.max_attempts = stoneridge.get_config_int('cloner', 'attempts')
        self.operating_systems = operating_systems
        self.netconfigs = netconfigs
        self.ldap = ldap
        self.sha = sha
        self.attempt = attempt

        if not os.path.exists(self.outroot):
            os.mkdir(self.outroot)

        root = stoneridge.get_config('cloner', 'root')
        if nightly:
            self.path = '/'.join([root, 'nightly', 'latest-mozilla-central'])
        else:
            self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)])

        logging.debug('host: %s' % (self.host,))
        logging.debug('path: %s' % (self.path,))
        logging.debug('nightly: %s' % (self.nightly,))
        logging.debug('srid: %s' % (self.srid,))
        logging.debug('output root: %s' % (self.outroot,))
        logging.debug('output directory: %s' % (self.outdir,))
        logging.debug('keep history: %s' % (self.keep,))
        logging.debug('max attempts: %s' % (self.max_attempts,))
        logging.debug('operating systems: %s' % (self.operating_systems,))
        logging.debug('netconfigs: %s' % (self.netconfigs,))
        logging.debug('ldap: %s' % (self.ldap,))
        logging.debug('sha: %s' % (self.sha,))
        logging.debug('attempt: %s' % (self.attempt,))

        self.prefix = ''

    def _gather_filelist(self, path):
        """Get the list of files available on our FTP server

        Returns: list of filenames relative to the path on the server
        """
        logging.debug('gathering files from ftp server')

        try:
            ftp = ftplib.FTP(self.host)
            ftp.login()
            ftp.cwd(path)
            files = ftp.nlst()
            ftp.quit()
        except:
            # We blanket-catch exceptions here, because we want the error
            # handling in the top level to take precedence for ANY problem that
            # happens while listing the directory. Logging helps us track down
            # unexpected errors that may occur.
            logging.exception('Unable to list files in %s' % (path,))
            return []

        logging.debug('files in %s: %s' % (path, files))
        return files

    def _build_dl_url(self, try_subdir, fname):
        """Create a download (https) URL for a particular file

        Returns: a URL string
        """
        logging.debug('creating download url for %s' % (fname,))
        remotefile = self.path
        if not self.nightly:
            remotefile = '/'.join([remotefile, try_subdir])
        remotefile = '/'.join([remotefile, fname])
        logging.debug('remote filename: %s' % (remotefile,))
        url = 'https://%s%s' % (self.host, remotefile)
        logging.debug('url: %s' % (url,))
        return url

    def _get_prefix(self, files):
        """Get the filename prefix that is common to all the files we'll need to
        download

        Returns: <prefix (string)>
        """
        logging.debug('getting filename prefix')
        prefixfile = [f for f in files if f.endswith('.checksums.asc')][-1]
        prefix = prefixfile.replace('.checksums.asc', '')
        prefix = prefix.rsplit('.', 1)[0] # Strip off the platform information
        logging.debug('filename prefix: %s' % (prefix,))
        return prefix

    def _ensure_outdir(self, platform):
        """Ensure the output directory for a platform exists
        """
        logging.debug('ensuring output directory for %s exists' % (platform,))
        if not os.path.exists(self.outdir):
            logging.debug('creating outdir %s' % (self.outdir,))
            os.mkdir(self.outdir)
        platdir = os.path.join(self.outdir, platform)
        logging.debug('platform directory: %s' % (platdir,))
        if not os.path.exists(platdir):
            logging.debug('creating platform directory %s' % (platdir,))
            os.mkdir(platdir)

    def _dl_to_file(self, url, outfile):
        """Download the file at <url> and save it to the file
        at <outfile>
        """
        logging.debug('downloading %s => %s' % (url, outfile))
        resp = requests.get(url, timeout=30000)
        with file(outfile, 'wb') as f:
            logging.debug('writing file contents')
            f.write(resp.content)

    def _dl_test_zip(self, try_subdir, archid, outdir):
        """Download the test zip for a particular architecture id (<archid>) and
        save it at <outdir>/tests.zip
        """
        logging.debug('downloading test zip for %s to %s' % (archid, outdir))
        srcfile = '%s.%s.tests.zip' % (self.prefix, archid)
        logging.debug('zip source filename: %s' % (srcfile,))
        url = self._build_dl_url(try_subdir, srcfile)
        outfile = os.path.join(self.outdir, outdir, 'tests.zip')
        logging.debug('zip dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

    def _clone_mac(self):
        """Clone the dmg and tests zip for the mac build
        """
        logging.debug('cloning mac build')
        self._ensure_outdir('mac')

        logging.debug('downloading firefox dmg')
        dmg = '%s.mac.dmg' % (self.prefix,)
        logging.debug('dmg source filename: %s' % (dmg,))
        url = self._build_dl_url(MAC_SUBDIRS[0], dmg)
        outfile = os.path.join(self.outdir, 'mac', 'firefox.dmg')
        logging.debug('dmg dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

        self._dl_test_zip(MAC_SUBDIRS[0], 'mac', 'mac')

    def _clone_linux(self):
        """Clone the .tar.bz2 and tests zip for both 32-bit and 64-bit linux
        builds
        """
        logging.debug('cloning linux builds')
        archids = ('i686', 'x86_64')
        outdirs = ('linux32', 'linux64')
        for archid, outdir, subdir in zip(archids, outdirs, LINUX_SUBDIRS):
            logging.debug('architecture: %s' % (archid,))
            logging.debug('outdir: %s' % (outdir,))
            self._ensure_outdir(outdir)

            logging.debug('downloading firefox tarball')
            srcfile = '%s.linux-%s.tar.bz2' % (self.prefix, archid)
            logging.debug('tarball source filename: %s' % (srcfile,))
            url = self._build_dl_url(subdir, srcfile)
            outfile = os.path.join(self.outdir, outdir, 'firefox.tar.bz2')
            logging.debug('tarball dest filename: %s' % (outfile,))
            self._dl_to_file(url, outfile)

            self._dl_test_zip(subdir, 'linux-%s' % (archid,), outdir)

    def _clone_win(self):
        """Clone the firefox zip and tests zip for both 32-bit and 64-bit
        windows builds
        """
        logging.debug('cloning windows build')
        self._ensure_outdir('win32')

        logging.debug('downloading firefox zip')
        srcfile = '%s.win32.zip' % (self.prefix,)
        logging.debug('zip source filename: %s' % (srcfile,))
        url = self._build_dl_url(WINDOWS_SUBDIRS[0], srcfile)
        outfile = os.path.join(self.outdir, 'win32', 'firefox.zip')
        logging.debug('zip dest filename: %s' % (outfile,))
        self._dl_to_file(url, outfile)

        self._dl_test_zip(WINDOWS_SUBDIRS[0], 'win32', 'win32')

    def _cleanup_old_directories(self):
        """We only keep around so many directories of historical firefoxen. This
        gets rid of ones we don't care about any more
        """
        logging.debug('cleaning up old directories')
        with stoneridge.cwd(self.outroot):
            listing = os.listdir('.')
            logging.debug('candidate files: %s' %  (listing,))

            # We want to make sure that we're not looking at anything that's not
            # a directory that may have somehow gotten into our directory. We
            # also need to ignore dotfiles.
            directories = [l for l in listing
                           if os.path.isdir(l) and not l.startswith('.')]
            logging.debug('directories: %s' % (directories,))

            # Find out when the directories were last modified, and sort the
            # list by that, so we can delete the oldest ones.
            times = [(d, os.stat(d).st_mtime) for d in directories]
            times.sort(key=lambda x: x[1])

            # Now we can figure out which directories to delete!
            delete_us = times[:-self.keep]
            logging.debug('directories to delete: %s' % (delete_us,))

            for d in delete_us:
                logging.debug('removing %s' % (d,))
                shutil.rmtree(d)

    def defer(self):
        args = ['srdeferrer.py',
                '--srid', self.srid,
                '--config', stoneridge.get_config_file(),
                '--log', '/dev/null',
                '--pidfile', tempfile.mktemp(),
                '--attempt', self.attempt + 1]

        if self.nightly:
            args.append('--nightly')
        else:
            args.extend(['--ldap', self.ldap])
            args.extend(['--sha', self.sha])
            for ops in self.operating_systems:
                args.append('--%s' % (ops,))
            for nc in self.netconfigs:
                args.append('--%s' % (nc,))

        stoneridge.run_process(*args)

    def exit_and_maybe_defer(self, deferred_message):
        next_attempt = self.attempt + 1
        if next_attempt > self.max_attempts:
            logging.error('Unable to get build results for %s after %s '
                    'attempts. Cancelling run.' %
                    (self.srid, self.max_attempts))
        else:
            logging.debug(deferred_message)
            self.defer()
        sys.exit(1)

    def run(self):
        files = self._gather_filelist(self.path)
        if not self.nightly:
            # For some ungodly reason, try builds have a different directory
            # structure than nightly builds, so we have to handle them
            # differently. Instead of all output being at the same level,
            # they are separated out by platform for try builds. Le sigh.
            subdirs = []
            dist_files = None
            if 'linux' in self.operating_systems:
                subdirs.extend(LINUX_SUBDIRS)
            if 'mac' in self.operating_systems:
                subdirs.extend(MAC_SUBDIRS)
            if 'windows' in self.operating_systems:
                subdirs.extend(WINDOWS_SUBDIRS)

            # Be reasonably sure the try run is complete, such that everything
            # is ready for us to download.
            for d in subdirs:
                if d not in files:
                    self.exit_and_maybe_defer(
                            'Run %s not available: retry later' % (d,))

            dist_path = '/'.join([self.path, subdirs[0]])
            dist_files = self._gather_filelist(dist_path)

            if not dist_files:
                # We didn't get any files listed, but we should have. Just drop
                # this run on the floor
                logging.error('No files found! Dropping srid %s' % (self.srid,))
                sys.exit(1)

            files = dist_files

        if not files:
            self.exit_and_maybe_defer(
                    'No files found for %s: retry later' % (self.srid,))

        self.prefix = self._get_prefix(files)

        # Make sure our output directory exists
        if not os.path.exists(self.outdir):
            logging.debug('creating output directory')
            os.mkdir(self.outdir)

        # Now download all the builds and test zipfiles
        if self.nightly or 'mac' in self.operating_systems:
            self._clone_mac()
        if self.nightly or 'linux' in self.operating_systems:
            self._clone_linux()
        if self.nightly or 'windows' in self.operating_systems:
            self._clone_win()

        self._cleanup_old_directories()

@stoneridge.main
def main():
    parser = stoneridge.ArgumentParser()
    parser.add_argument('--nightly', dest='nightly', action='store_true',
            default=False)
    parser.add_argument('--srid', dest='srid', required=True)
    for ops in stoneridge.OPERATING_SYSTEMS:
        parser.add_argument('--%s' % (ops,), dest='operating_systems',
                action='append_const', const=ops, default=[])
    for nc in stoneridge.NETCONFIGS:
        parser.add_argument('--%s' % (nc,), dest='netconfigs',
                action='append_const', const=nc, default=[])
    parser.add_argument('--attempt', dest='attempt', required=True, type=int)
    parser.add_argument('--ldap', dest='ldap', default='')
    parser.add_argument('--sha', dest='sha', default='')
    args = parser.parse_args()

    cloner = StoneRidgeCloner(args.nightly, args.srid, args.operating_systems,
            args.netconfigs, args.ldap, args.sha, args.attempt)
    cloner.run()
<|endoftext|>


<commit_before># coding: utf-8

import datetime

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.mail import EmailMessage
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from django.db import models
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.translation import pgettext_lazy as _

from email_confirm_la import signals
from email_confirm_la.conf import configs
from email_confirm_la.compat import GenericForeignKey
from email_confirm_la.exceptions import ExpiredError
from email_confirm_la.utils import generate_random_token

class EmailConfirmationManager(models.Manager):

    def verify_email_for_object(self, email, content_object, email_field_name='email'):
        """
        Create an email confirmation for `content_object` and send a confirmation mail.

        The email will be directly saved to `content_object.email_field_name` when `is_primary` and `skip_verify` both are true.
        """

        confirmation_key = generate_random_token([str(content_object.__class__.__name__), str(content_object.id), str(email_field_name), email, ])

        try:
            confirmation = EmailConfirmation()
            confirmation.content_object = content_object
            confirmation.email_field_name = email_field_name
            confirmation.email = email
            confirmation.confirmation_key = confirmation_key
            confirmation.save()
        except IntegrityError:
            confirmation = EmailConfirmation.objects.get_for_object(content_object, email_field_name)
            confirmation.email = email
            confirmation.confirmation_key = confirmation_key
            confirmation.save(update_fields=['email', 'confirmation_key'])

        confirmation.send()

        return confirmation

    def get_unverified_email_for_object(self, content_object, email_field_name='email'):
        try:
            confirmation = EmailConfirmation.objects.get_for_object(content_object, email_field_name)
        except EmailConfirmation.DoesNotExist:
            unverified_email = None
        else:
            unverified_email = confirmation.email

        return unverified_email

    def get_for_object(self, content_object, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object)
        confirmation = EmailConfirmation.objects.get(content_type=content_type, object_id=content_object.id, email_field_name=email_field_name)

        return confirmation

    def get_queryset_for_object(self, content_object, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object)
        queryset = EmailConfirmation.objects.filter(content_type=content_type, object_id=content_object.id, email_field_name=email_field_name)

        return queryset

    def get_for_email(self, email, content_object_model, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object_model)
        confirmation = EmailConfirmation.objects.get(content_type=content_type, email_field_name=email_field_name, email=email)

        return confirmation

class EmailConfirmation(models.Model):
    """
    Once an email is confirmed, it will be delete from this model. In other words, there are only unconfirmed emails in the database.
    """

    ExpiredError = ExpiredError

    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    email_field_name = models.CharField(verbose_name=_('ec_la', 'Email field name'), max_length=32, default='email')
    email = models.EmailField(verbose_name=_('ec_la', 'Email'))
    confirmation_key = models.CharField(verbose_name=_('ec_la', 'Confirmation_key'), max_length=64, unique=True)
    send_at = models.DateTimeField(null=True, blank=True, db_index=True)
    # confirmed_at = models.DateTimeField(null=True, blank=True)

    objects = EmailConfirmationManager()

    class Meta:
        verbose_name = _('ec_la', 'Email confirmation')
        verbose_name_plural = _('ec_la', 'Email confirmation')
        unique_together = (('content_type', 'email_field_name', 'object_id', 'email'), )
        unique_together = (('content_type', 'object_id', 'email_field_name'), )

    def __repr__(self):
        return '<EmailConfirmation {0}>'.format(self.email)

    def __str__(self):
        return 'Confirmation for {0}'.format(self.email)

    def __unicode__(self):
        return 'Confirmation for {0}'.format(self.email)

    def send(self, template_context=None):
        default_template_context = dict(configs.EMAIL_CONFIRM_LA_TEMPLATE_CONTEXT)
        default_template_context['email_confirmation'] = self

        if isinstance(template_context, dict):
            template_context = dict(default_template_context.items() + template_context.items())  # merge dictionaries
        else:
            template_context = default_template_context

        subject = render_to_string('email_confirm_la/email/email_confirmation_subject.txt', template_context)
        subject = ''.join(subject.splitlines()).strip()  # remove unnecessary line breaks
        body = render_to_string('email_confirm_la/email/email_confirmation_message.html', template_context)
        message = EmailMessage(subject, body, settings.DEFAULT_FROM_EMAIL, [self.email, ])
        message.content_subtype = 'html'
        message.send()

        self.send_at = timezone.now()
        self.save(update_fields=('send_at', ))

        signals.post_email_confirmation_send.send(
            sender=self.__class__,
            confirmation=self,
        )

    def get_confirmation_url(self, full=True):
        url_reverse_name = configs.EMAIL_CONFIRM_LA_CONFIRM_URL_REVERSE_NAME
        url = reverse(url_reverse_name, kwargs={'confirmation_key': self.confirmation_key})
        if full:
            confirmation_url = '{0}://{1}{2}'.format(configs.EMAIL_CONFIRM_LA_HTTP_PROTOCOL, configs.EMAIL_CONFIRM_LA_DOMAIN, url)
        else:
            confirmation_url = url

        return confirmation_url

    def confirm(self, ignore_expiration=False, save_to_content_object=True):
        if not ignore_expiration and self.is_expired:
            raise ExpiredError()

        if save_to_content_object:
            setattr(self.content_object, self.email_field_name, self.email)
            self.content_object.save(update_fields=(self.email_field_name, ))

        signals.post_email_confirmation_confirm.send(
            sender=self.__class__,
            confirmation=self,
            save_to_content_object=save_to_content_object,
        )

    def clean(self):
        """
        delete all confirmations for the same content_object and the same field
        """

        EmailConfirmation.objects.filter(content_type=self.content_type, object_id=self.object_id, email_field_name=self.email_field_name).delete()

    @property
    def is_expired(self):
        if not self.send_at:
            return False

        expiration_time = self.send_at + datetime.timedelta(seconds=configs.EMAIL_CONFIRM_LA_CONFIRM_EXPIRE_SEC)

        return expiration_time <= timezone.now()
<commit_msg>use ugettext_lazy instead of pgettext_lazy<commit_after># coding: utf-8

import datetime

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.mail import EmailMessage
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from django.db import models
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from email_confirm_la import signals
from email_confirm_la.conf import configs
from email_confirm_la.compat import GenericForeignKey
from email_confirm_la.exceptions import ExpiredError
from email_confirm_la.utils import generate_random_token

class EmailConfirmationManager(models.Manager):

    def verify_email_for_object(self, email, content_object, email_field_name='email'):
        """
        Create an email confirmation for `content_object` and send a confirmation mail.

        The email will be directly saved to `content_object.email_field_name` when `is_primary` and `skip_verify` both are true.
        """

        confirmation_key = generate_random_token([str(content_object.__class__.__name__), str(content_object.id), str(email_field_name), email, ])

        try:
            confirmation = EmailConfirmation()
            confirmation.content_object = content_object
            confirmation.email_field_name = email_field_name
            confirmation.email = email
            confirmation.confirmation_key = confirmation_key
            confirmation.save()
        except IntegrityError:
            confirmation = EmailConfirmation.objects.get_for_object(content_object, email_field_name)
            confirmation.email = email
            confirmation.confirmation_key = confirmation_key
            confirmation.save(update_fields=['email', 'confirmation_key'])

        confirmation.send()

        return confirmation

    def get_unverified_email_for_object(self, content_object, email_field_name='email'):
        try:
            confirmation = EmailConfirmation.objects.get_for_object(content_object, email_field_name)
        except EmailConfirmation.DoesNotExist:
            unverified_email = ''
        else:
            unverified_email = confirmation.email

        return unverified_email

    def get_for_object(self, content_object, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object)
        confirmation = EmailConfirmation.objects.get(content_type=content_type, object_id=content_object.id, email_field_name=email_field_name)

        return confirmation

    def get_queryset_for_object(self, content_object, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object)
        queryset = EmailConfirmation.objects.filter(content_type=content_type, object_id=content_object.id, email_field_name=email_field_name)

        return queryset

    def get_for_email(self, email, content_object_model, email_field_name='email'):
        content_type = ContentType.objects.get_for_model(content_object_model)
        confirmation = EmailConfirmation.objects.get(content_type=content_type, email_field_name=email_field_name, email=email)

        return confirmation

class EmailConfirmation(models.Model):
    """
    Once an email is confirmed, it will be delete from this table. In other words, there are only unconfirmed emails in the database.
    """

    ExpiredError = ExpiredError

    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    email_field_name = models.CharField(verbose_name=_(u'Email field name'), max_length=32, default='email')
    email = models.EmailField(verbose_name=_(u'Email'), db_index=True)
    confirmation_key = models.CharField(verbose_name=_(u'Confirmation_key'), max_length=64, unique=True)
    send_at = models.DateTimeField(null=True, blank=True, db_index=True)

    objects = EmailConfirmationManager()

    class Meta:
        verbose_name = _(u'Email confirmation')
        verbose_name_plural = _(u'Email confirmation')
        unique_together = (('content_type', 'object_id', 'email_field_name'), )

    def __repr__(self):
        return '<EmailConfirmation {0}>'.format(self.email)

    def __str__(self):
        return 'Confirmation for {0}'.format(self.email)

    def __unicode__(self):
        return u'Confirmation for {0}'.format(self.email)

    def send(self, template_context=None):
        default_template_context = dict(configs.EMAIL_CONFIRM_LA_TEMPLATE_CONTEXT)
        default_template_context['email_confirmation'] = self

        if isinstance(template_context, dict):
            template_context = dict(default_template_context.items() + template_context.items())  # merge dictionaries
        else:
            template_context = default_template_context

        subject = render_to_string('email_confirm_la/email/email_confirmation_subject.txt', template_context)
        subject = ''.join(subject.splitlines()).strip()  # remove unnecessary line breaks
        body = render_to_string('email_confirm_la/email/email_confirmation_message.html', template_context)
        message = EmailMessage(subject, body, settings.DEFAULT_FROM_EMAIL, [self.email, ])
        message.content_subtype = 'html'
        message.send()

        self.send_at = timezone.now()
        self.save(update_fields=('send_at', ))

        signals.post_email_confirmation_send.send(
            sender=self.__class__,
            confirmation=self,
        )

    def get_confirmation_url(self, full=True):
        url_reverse_name = configs.EMAIL_CONFIRM_LA_CONFIRM_URL_REVERSE_NAME
        url = reverse(url_reverse_name, kwargs={'confirmation_key': self.confirmation_key})
        if full:
            confirmation_url = '{0}://{1}{2}'.format(configs.EMAIL_CONFIRM_LA_HTTP_PROTOCOL, configs.EMAIL_CONFIRM_LA_DOMAIN, url)
        else:
            confirmation_url = url

        return confirmation_url

    def confirm(self, ignore_expiration=False, save_to_content_object=True):
        if not ignore_expiration and self.is_expired:
            raise ExpiredError()

        if save_to_content_object:
            setattr(self.content_object, self.email_field_name, self.email)
            self.content_object.save(update_fields=(self.email_field_name, ))

        signals.post_email_confirmation_confirm.send(
            sender=self.__class__,
            confirmation=self,
            save_to_content_object=save_to_content_object,
        )

    def clean(self):
        """
        delete all confirmations for the same content_object and the same field
        """

        EmailConfirmation.objects.filter(content_type=self.content_type, object_id=self.object_id, email_field_name=self.email_field_name).delete()

    @property
    def is_expired(self):
        if not self.send_at:
            return False

        expiration_time = self.send_at + datetime.timedelta(seconds=configs.EMAIL_CONFIRM_LA_CONFIRM_EXPIRE_SEC)

        return expiration_time <= timezone.now()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from utils import utils, inspector, admin

# http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - The link to http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf
# is incorrect. See `REPORT_URL_MAPPING`
# - Fix all links in `BLACKLIST_REPORT_URLS`

AUDIT_REPORTS_URL = "http://www.fmc.gov/about/oig_audits_evaluations_and_reviews.aspx"
SEMIANNUAL_REPORTS_URL = "http://www.fmc.gov/about/oig_semiannual_reports.aspx"

REPORT_URL_MAPPING = {
  "http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf":
  "http://www.fmc.gov/assets/1/Page/OR10-03_UserFeeCalculations.pdf",
}

BLACKLIST_REPORT_URLS = [
  "http://www.fmc.gov/UserFiles/pages/File/FY_08_Audited_Financial_Statements_09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_09-01A.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_OR09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-02.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-03.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-04.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_A09-05.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-06.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-07.pdf",
]

REPORT_PUBLISHED_MAP = {
  "508letterOIG2016": datetime.datetime(2016, 5, 27),
  "A17-01": datetime.datetime(2016, 11, 9),
  "A17-02": datetime.datetime(2016, 11, 8),
  "A16-01": datetime.datetime(2015, 11, 10),
  "A16-02": datetime.datetime(2015, 11, 12),
  "A16-03": datetime.datetime(2016, 9, 9),
  "A15-01": datetime.datetime(2014, 11, 14),
  "A15-01A": datetime.datetime(2014, 12, 31),
  "A15-02": datetime.datetime(2014, 11, 14),
  "A15-03": datetime.datetime(2014, 11, 14),
  "A15-04": datetime.datetime(2015, 3, 10),
  "A15-05": datetime.datetime(2015, 9, 25),
  "A14-01": datetime.datetime(2013, 12, 12),
  "A14-01A": datetime.datetime(2014, 1, 31),
  "A14-02": datetime.datetime(2014, 1, 3),
  "A13-01": datetime.datetime(2012, 11, 6),
  "A13-02": datetime.datetime(2012, 12, 6),
  "A13-03": datetime.datetime(2012, 12, 21),
  "A13-04": datetime.datetime(2012, 12, 14),
  "A13-04A": datetime.datetime(2013, 3, 15),
  "A12-01": datetime.datetime(2011, 11, 9),
  "A12-02": datetime.datetime(2012, 1, 17),
  "OR12-01": datetime.datetime(2012, 3, 2),
  "A12-01A": datetime.datetime(2012, 3, 5),
  "OR12-02": datetime.datetime(2012, 7, 17),
  "OR11-02": datetime.datetime(2011, 9, 30),
  "OR11-01": datetime.datetime(2011, 3, 16),
  "A11-02A": datetime.datetime(2011, 1, 31),
  "A11-02": datetime.datetime(2010, 11, 10),
  "A11-01A": datetime.datetime(2010, 11, 8),
  "A11-01": datetime.datetime(2010, 12, 15),
  "OR10-04": datetime.datetime(2010, 8, 6),
  "OR10-03": datetime.datetime(2010, 5, 27),
  "OR10-02": datetime.datetime(2010, 5, 14),
  "A10-01": datetime.datetime(2009, 11, 6),
  "A10-01A": datetime.datetime(2010, 3, 2),
  "OR10-01": datetime.datetime(2010, 3, 4),
  "A10-02": datetime.datetime(2010, 1, 28),
  "A10-03": datetime.datetime(2010, 3, 1),
  "A09-01": datetime.datetime(2008, 11, 6),
  "A09-01A": datetime.datetime(2009, 1, 15),
  "OR09-01": datetime.datetime(2009, 1, 12),
  "A09-02": datetime.datetime(2009, 2, 6),
  "A09-03": datetime.datetime(2009, 7, 7),
  "A09-04": datetime.datetime(2009, 6, 30),
  "A09-05": datetime.datetime(2009, 7, 20),
  "A09-06": datetime.datetime(2009, 7, 28),
  "A09-07": datetime.datetime(2009, 8, 21),
  "A08-01": datetime.datetime(2007, 11, 16),
  "A08-02": datetime.datetime(2007, 11, 6),
  "A08-02A": datetime.datetime(2007, 12, 12),
  "A08-03": datetime.datetime(2008, 1, 23),
  "A08-04": datetime.datetime(2008, 3, 18),
  "A08-05": datetime.datetime(2008, 8, 29),
  "A08-06": datetime.datetime(2008, 9, 10),
  "A08-07": datetime.datetime(2008, 9, 22),
  "A08-08": datetime.datetime(2008, 9, 22),
  "A07-01": datetime.datetime(2006, 11, 13),
  "OR07-01": datetime.datetime(2007, 1, 19),
  "A07-02": datetime.datetime(2007, 5, 4),
  "OR07-02": datetime.datetime(2007, 6, 29),
  "A06-01": datetime.datetime(2006, 3, 30),
  "OR06-01": datetime.datetime(2006, 8, 22),
  "A06-02": datetime.datetime(2006, 8, 1),
  "A06-04": datetime.datetime(2006, 10, 2),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_REPORTS_URL)
  results = doc.select("table tr")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (audits)")
  for result in results:
    if result.th:
      # Skip the header row
      continue
    report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull historical audits
  audit_year_links = doc.select("div.col-2-3 ul li a")
  for year_link in audit_year_links:
    audit_year_url = urljoin(AUDIT_REPORTS_URL, year_link.get('href'))
    doc = utils.beautifulsoup_from_url(audit_year_url)
    results = doc.select("table tr")
    if not results:
      # Grab results other than first and last (header and extra links)
      results = doc.select("div.col-2-2 ul")[1:-1]
    if not results:
      raise inspector.NoReportsFoundError("Federal Maritime Commission (%s)" % audit_year_url)
    for result in results:
      if result.th:
        # Skip the header row
        continue
      report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div.col-2-2 p a") + doc.select("div.col-2-2 li a")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (semiannual reports)")
  for result in results:
    report = report_from(result.parent, AUDIT_REPORTS_URL, report_type='semiannual_report', year_range=year_range)
    if report:
      inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  report_url = urljoin(landing_url, link.get('href'))
  title = link.text

  if report_url in REPORT_URL_MAPPING:
    report_url = REPORT_URL_MAPPING[report_url]

  if report_url in BLACKLIST_REPORT_URLS:
    return

  try:
    report_id = result.select("td")[0].text
  except IndexError:
    try:
      report_id = result.select("li")[0].text
    except IndexError:
      report_filename = report_url.split("/")[-1]
      report_id, _ = os.path.splitext(report_filename)

  published_on = None
  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  if not published_on:
    try:
      published_on_text = title.split("-")[-1].strip()
      published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')
    except ValueError:
      pass

  if not published_on:
    admin.log_no_date("fmc", report_id, title, report_url)
    return

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'fmc',
    'inspector_url': 'http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx',
    'agency': 'fmc',
    'agency_name': 'Federal Maritime Commission',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[fmc] Add date<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from utils import utils, inspector, admin

# http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - The link to http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf
# is incorrect. See `REPORT_URL_MAPPING`
# - Fix all links in `BLACKLIST_REPORT_URLS`

AUDIT_REPORTS_URL = "http://www.fmc.gov/about/oig_audits_evaluations_and_reviews.aspx"
SEMIANNUAL_REPORTS_URL = "http://www.fmc.gov/about/oig_semiannual_reports.aspx"

REPORT_URL_MAPPING = {
  "http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf":
  "http://www.fmc.gov/assets/1/Page/OR10-03_UserFeeCalculations.pdf",
}

BLACKLIST_REPORT_URLS = [
  "http://www.fmc.gov/UserFiles/pages/File/FY_08_Audited_Financial_Statements_09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_09-01A.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_OR09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-02.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-03.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-04.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_A09-05.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-06.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-07.pdf",
]

REPORT_PUBLISHED_MAP = {
  "AgencylettertoCongress": datetime.datetime(2016, 11, 28),
  "508letterOIG2016": datetime.datetime(2016, 5, 27),
  "A17-01": datetime.datetime(2016, 11, 9),
  "A17-02": datetime.datetime(2016, 11, 8),
  "A16-01": datetime.datetime(2015, 11, 10),
  "A16-02": datetime.datetime(2015, 11, 12),
  "A16-03": datetime.datetime(2016, 9, 9),
  "A15-01": datetime.datetime(2014, 11, 14),
  "A15-01A": datetime.datetime(2014, 12, 31),
  "A15-02": datetime.datetime(2014, 11, 14),
  "A15-03": datetime.datetime(2014, 11, 14),
  "A15-04": datetime.datetime(2015, 3, 10),
  "A15-05": datetime.datetime(2015, 9, 25),
  "A14-01": datetime.datetime(2013, 12, 12),
  "A14-01A": datetime.datetime(2014, 1, 31),
  "A14-02": datetime.datetime(2014, 1, 3),
  "A13-01": datetime.datetime(2012, 11, 6),
  "A13-02": datetime.datetime(2012, 12, 6),
  "A13-03": datetime.datetime(2012, 12, 21),
  "A13-04": datetime.datetime(2012, 12, 14),
  "A13-04A": datetime.datetime(2013, 3, 15),
  "A12-01": datetime.datetime(2011, 11, 9),
  "A12-02": datetime.datetime(2012, 1, 17),
  "OR12-01": datetime.datetime(2012, 3, 2),
  "A12-01A": datetime.datetime(2012, 3, 5),
  "OR12-02": datetime.datetime(2012, 7, 17),
  "OR11-02": datetime.datetime(2011, 9, 30),
  "OR11-01": datetime.datetime(2011, 3, 16),
  "A11-02A": datetime.datetime(2011, 1, 31),
  "A11-02": datetime.datetime(2010, 11, 10),
  "A11-01A": datetime.datetime(2010, 11, 8),
  "A11-01": datetime.datetime(2010, 12, 15),
  "OR10-04": datetime.datetime(2010, 8, 6),
  "OR10-03": datetime.datetime(2010, 5, 27),
  "OR10-02": datetime.datetime(2010, 5, 14),
  "A10-01": datetime.datetime(2009, 11, 6),
  "A10-01A": datetime.datetime(2010, 3, 2),
  "OR10-01": datetime.datetime(2010, 3, 4),
  "A10-02": datetime.datetime(2010, 1, 28),
  "A10-03": datetime.datetime(2010, 3, 1),
  "A09-01": datetime.datetime(2008, 11, 6),
  "A09-01A": datetime.datetime(2009, 1, 15),
  "OR09-01": datetime.datetime(2009, 1, 12),
  "A09-02": datetime.datetime(2009, 2, 6),
  "A09-03": datetime.datetime(2009, 7, 7),
  "A09-04": datetime.datetime(2009, 6, 30),
  "A09-05": datetime.datetime(2009, 7, 20),
  "A09-06": datetime.datetime(2009, 7, 28),
  "A09-07": datetime.datetime(2009, 8, 21),
  "A08-01": datetime.datetime(2007, 11, 16),
  "A08-02": datetime.datetime(2007, 11, 6),
  "A08-02A": datetime.datetime(2007, 12, 12),
  "A08-03": datetime.datetime(2008, 1, 23),
  "A08-04": datetime.datetime(2008, 3, 18),
  "A08-05": datetime.datetime(2008, 8, 29),
  "A08-06": datetime.datetime(2008, 9, 10),
  "A08-07": datetime.datetime(2008, 9, 22),
  "A08-08": datetime.datetime(2008, 9, 22),
  "A07-01": datetime.datetime(2006, 11, 13),
  "OR07-01": datetime.datetime(2007, 1, 19),
  "A07-02": datetime.datetime(2007, 5, 4),
  "OR07-02": datetime.datetime(2007, 6, 29),
  "A06-01": datetime.datetime(2006, 3, 30),
  "OR06-01": datetime.datetime(2006, 8, 22),
  "A06-02": datetime.datetime(2006, 8, 1),
  "A06-04": datetime.datetime(2006, 10, 2),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_REPORTS_URL)
  results = doc.select("table tr")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (audits)")
  for result in results:
    if result.th:
      # Skip the header row
      continue
    report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull historical audits
  audit_year_links = doc.select("div.col-2-3 ul li a")
  for year_link in audit_year_links:
    audit_year_url = urljoin(AUDIT_REPORTS_URL, year_link.get('href'))
    doc = utils.beautifulsoup_from_url(audit_year_url)
    results = doc.select("table tr")
    if not results:
      # Grab results other than first and last (header and extra links)
      results = doc.select("div.col-2-2 ul")[1:-1]
    if not results:
      raise inspector.NoReportsFoundError("Federal Maritime Commission (%s)" % audit_year_url)
    for result in results:
      if result.th:
        # Skip the header row
        continue
      report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div.col-2-2 p a") + doc.select("div.col-2-2 li a")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (semiannual reports)")
  for result in results:
    report = report_from(result.parent, AUDIT_REPORTS_URL, report_type='semiannual_report', year_range=year_range)
    if report:
      inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  report_url = urljoin(landing_url, link.get('href'))
  title = link.text

  if report_url in REPORT_URL_MAPPING:
    report_url = REPORT_URL_MAPPING[report_url]

  if report_url in BLACKLIST_REPORT_URLS:
    return

  try:
    report_id = result.select("td")[0].text
  except IndexError:
    try:
      report_id = result.select("li")[0].text
    except IndexError:
      report_filename = report_url.split("/")[-1]
      report_id, _ = os.path.splitext(report_filename)

  published_on = None
  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  if not published_on:
    try:
      published_on_text = title.split("-")[-1].strip()
      published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')
    except ValueError:
      pass

  if not published_on:
    admin.log_no_date("fmc", report_id, title, report_url)
    return

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'fmc',
    'inspector_url': 'http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx',
    'agency': 'fmc',
    'agency_name': 'Federal Maritime Commission',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>


<commit_before># Controls the rotating platform
#
# The servo rotates the platform through a friction wheel.
# This wheel can slip when it is blocked in any way as a protection.
#
# A connected webcam looks down on the platform to control position.
import datetime
import requests

import picamera

import serial

import time

import sys

ser = serial.Serial(sys.argv[1],9600)

camera = picamera.PiCamera()
#camera.resolution = (2592, 1944)
#camera.shutter_speed = camera.exposure_speed
#camera.exposure_mode = 'off'
#g = camera.awb_gains
#camera.awb_mode = 'off'
#camera.awb_gains = g

print("testing")

counter = 0

def turn_and_capture(counter):
    r = requests.get('http://digitalfoodone.appspot.com/controlrequest')
    print(r.status_code)
    print(r.headers['content-type'])
    print(r.encoding)
    print(r.text)
    data = r.json()
    print(data)
    upload_url = data['upload_url']

    camera.capture('my_img.jpg')

    files = {'file':open('my_img.jpg','rb')}
    data = {
        'green points':'abc',
        'average_x':10,
        'average_y':110,
    }
    resp = requests.post(upload_url,files=files, data=data)
    print(resp)

turn_and_capture(counter)
<commit_msg>image analysis start<commit_after># Controls the rotating platform
#
# The servo rotates the platform through a friction wheel.
# This wheel can slip when it is blocked in any way as a protection.
#
# A connected webcam looks down on the platform to control position.
import datetime
import requests

import picamera

import serial

import time

import sys

import PIL

ser = serial.Serial(sys.argv[1],9600)

camera = picamera.PiCamera()
#camera.resolution = (2592, 1944)
#camera.shutter_speed = camera.exposure_speed
#camera.exposure_mode = 'off'
#g = camera.awb_gains
#camera.awb_mode = 'off'
#camera.awb_gains = g

print("testing")

counter = 0

def turn_and_capture(counter):
    r = requests.get('http://digitalfoodone.appspot.com/controlrequest')
    print(r.status_code)
    print(r.headers['content-type'])
    print(r.encoding)
    print(r.text)
    data = r.json()
    print(data)
    upload_url = data['upload_url']

    camera.capture('my_img.jpg')

    files = {'file':open('my_img.jpg','rb')}

    im = Image.open('my_img.jpg')
    pixels = list(im.getdata())
    width, height = im.size
    pixels = [pixels[i * width:(i + 1) * width] for i in xrange(height)]
    red_points = []
    for x in range(0,width, 2):
        for y in range(0, height, 2):
            if(pixels[x][y][0] > 150):
                red_points.append({
                    'x':x,
                    'y':y
                })

    data = {
        'analysis_data': {
            'red points':'red_points',
            'average_x':10,
            'average_y':110,
        }
    }
    resp = requests.post(upload_url,files=files, data=data)
    print(resp)

turn_and_capture(counter)
<|endoftext|>


<commit_before>"""Extends BaseHTTPRequestHandler with SSL certificate generation."""
import logging
import socket

import certutils

class SslHandshakeHandler:
  """Handles Server Name Indication (SNI) using dummy certs."""

  def setup(self):
    """Sets up connection providing the certificate to the client."""
    # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
    context = certutils.get_ssl_context()
    def handle_servername(connection):
      """A SNI callback that happens during do_handshake()."""
      try:
        host = connection.get_servername()
        if host:
          cert_str = (
              self.server.http_archive_fetch.http_archive.get_certificate(host))
          new_context = certutils.get_ssl_context()
          cert = certutils.load_cert(cert_str)
          new_context.use_certificate(cert)
          new_context.use_privatekey_file(self.server.ca_cert_path)
          connection.set_context(new_context)
          return new_context
        # else: fail with 'no shared cipher'
      except Exception, e:
        # Do not leak any exceptions or else openssl crashes.
        logging.error('Exception in SNI handler', e)

    context.set_tlsext_servername_callback(handle_servername)
    self.connection = certutils.get_ssl_connection(context, self.connection)
    self.connection.set_accept_state()
    try:
      self.connection.do_handshake()
    except certutils.Error, v:
      host = self.connection.get_servername()
      if not host:
        logging.error('Dropping request without SNI')
        return ''
      raise certutils.Error('SSL handshake error %s: %s' % (host, str(v)))

    # Re-wrap the read/write streams with our new connection.
    self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize,
                                    close=False)
    self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize,
                                    close=False)

  def finish(self):
    self.connection.shutdown()
    self.connection.close()

def wrap_handler(handler_class):
  """Wraps a BaseHTTPHandler wtih SSL MITM certificates."""
  if certutils.openssl_import_error:
    raise certutils.openssl_import_error

  class WrappedHandler(SslHandshakeHandler, handler_class):

    def setup(self):
      handler_class.setup(self)
      SslHandshakeHandler.setup(self)

    def finish(self):
      handler_class.finish(self)
      SslHandshakeHandler.finish(self)
  return WrappedHandler

<commit_msg>fixing whitespace<commit_after>"""Extends BaseHTTPRequestHandler with SSL certificate generation."""
import logging
import socket

import certutils

class SslHandshakeHandler:
  """Handles Server Name Indication (SNI) using dummy certs."""

  def setup(self):
    """Sets up connection providing the certificate to the client."""
    # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
    context = certutils.get_ssl_context()
    def handle_servername(connection):
      """A SNI callback that happens during do_handshake()."""
      try:
        host = connection.get_servername()
        if host:
          cert_str = (
              self.server.http_archive_fetch.http_archive.get_certificate(host))
          new_context = certutils.get_ssl_context()
          cert = certutils.load_cert(cert_str)
          new_context.use_certificate(cert)
          new_context.use_privatekey_file(self.server.ca_cert_path)
          connection.set_context(new_context)
          return new_context
        # else: fail with 'no shared cipher'
      except Exception, e:
        # Do not leak any exceptions or else openssl crashes.
        logging.error('Exception in SNI handler', e)

    context.set_tlsext_servername_callback(handle_servername)
    self.connection = certutils.get_ssl_connection(context, self.connection)
    self.connection.set_accept_state()
    try:
      self.connection.do_handshake()
    except certutils.Error, v:
      host = self.connection.get_servername()
      if not host:
        logging.error('Dropping request without SNI')
        return ''
      raise certutils.Error('SSL handshake error %s: %s' % (host, str(v)))

    # Re-wrap the read/write streams with our new connection.
    self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize,
                                    close=False)
    self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize,
                                    close=False)

  def finish(self):
    self.connection.shutdown()
    self.connection.close()

def wrap_handler(handler_class):
  """Wraps a BaseHTTPHandler wtih SSL MITM certificates."""
  if certutils.openssl_import_error:
    raise certutils.openssl_import_error

  class WrappedHandler(SslHandshakeHandler, handler_class):

    def setup(self):
      handler_class.setup(self)
      SslHandshakeHandler.setup(self)

    def finish(self):
      handler_class.finish(self)
      SslHandshakeHandler.finish(self)
  return WrappedHandler
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# base.py
#
# Django settings for inventory project.
#

import os
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

from inventory.apps.items.settings import *
from inventory.apps.login.settings import *

class IPList(list):

    def __init__(self, ips):
        try:
            #http://software.inl.fr/trac/wiki/IPy
            #ubuntu: apt-get install python-ipy
            from IPy import IP
            for ip in ips:
                self.append(IP(ip))
        except ImportError:
            pass

    def __contains__(self, ip):
        try:
            for net in self:
                if ip in net:
                    return True
        except:
            pass

        return False

DEBUG = False

ADMINS = (
    ('Sysadmins', 'carl.nobile@gmail.com'),
    )

ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

SITE_NAME = "TetraSys Inventory"

# Where is the 'website' directory with settings dir, apps, urls.py, etc. are.
SITE_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

# Where is the root of the site? This can be a root-relative URL.
SITE_URL = '/'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = os.path.join(SITE_ROOT, 'media/')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = os.path.join(SITE_URL, 'media/')

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = os.path.join(SITE_ROOT, 'static/')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = os.path.join(SITE_URL, 'static/')

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'dev/'),
    )

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    #'django.contrib.staticfiles.finders.DefaultStorageFinder',
    )

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'bjwykl6a7km26!0bsx%$v8g#s=+s5-(v2&d0^r8tl5++4ip$u4'

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend', # default
    'guardian.backends.ObjectPermissionBackend',
    )

# Guardian requirement
ANONYMOUS_USER_ID = -1

MIDDLEWARE_CLASSES = [
    # UpdateCacheMiddleware must be first on the list
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware', # This must be last
    ]

CACHES = {
    'default': {
        'BACKEND': 'redis_cache.RedisCache',
        'LOCATION': '/var/run/redis/redis.sock',
        },
    }

SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

ROOT_URLCONF = 'inventory.urls'

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'inventory.wsgi.application'

AUTH_USER_MODEL = 'accounts.User'

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.admindocs',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'dcolumn.dcolumns',
    'rest_framework',
    'oauth2_provider',
    #'guardian',
    'inventory.common',
    'inventory.accounts',
    'inventory.oauth2',
    'inventory.projects',
    'inventory.regions',
    'inventory.suppliers',
    'inventory.categories',
    'inventory.maintenance',
    #'inventory.objects',
    # Will be removed over time.
    'inventory.apps.items',
    'inventory.apps.login',
    'inventory.apps.reports',
    ]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(SITE_ROOT, 'templates'),
            ],
#        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                ],
            'debug': DEBUG,
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
                #'django.template.loaders.eggs.Loader',
                ],
        },
    },
]

OAUTH2_PROVIDER = {
    # This is the list of available scopes.
    'SCOPES': {
        'read': 'Read scope',
        'write': 'Write scope',
        'groups': 'Access to your groups'
        }
    }

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
        ),
    'DEFAULT_PARSER_CLASSES': (
        'rest_framework.parsers.JSONParser',
        'rest_framework_xml.parsers.XMLParser',
        'rest_framework_yaml.parsers.YAMLParser',
        ),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.BrowsableAPIRenderer',
        'rest_framework.renderers.JSONRenderer',
        'rest_framework_xml.renderers.XMLRenderer',
        'rest_framework_yaml.renderers.YAMLRenderer',
        ),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework.authentication.SessionAuthentication',
        'oauth2_provider.ext.rest_framework.OAuth2Authentication',
        ),
    'TEST_REQUEST_DEFAULT_FORMAT': 'json',
    'TEST_REQUEST_RENDERER_CLASSES': (
        'rest_framework.renderers.MultiPartRenderer',
        'rest_framework.renderers.JSONRenderer',
        'rest_framework.renderers.TemplateHTMLRenderer'
        )
    }

# A sample logging configuration. The only tangible logging performed by this
# configuration is to send an email to the site admins on every HTTP 500 error
# when DEBUG=False. See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOG_DIR = os.path.abspath(os.path.join(SITE_ROOT, '..', 'logs'))
not os.path.isdir(LOG_DIR) and os.mkdir(LOG_DIR, 0775)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': "%(asctime)s %(levelname)s %(module)s %(funcName)s "
            "[line:%(lineno)d] %(message)s"
            },
        'simple': {
            'format': '%(asctime)s %(levelname)s %(message)s'
            },
        },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
            },
        },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
            },
        'console': {
            'level': 'WARNING',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
            },
        # Remove this one in the future.
        'inventory_file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'DEBUG',
            'formatter': 'verbose',
            'filename': '/dev/null',
            'maxBytes': 50000000,  # 50 Meg bytes
            'backupCount': 5,
            },
        'api_file': {
            'class': 'inventory.common.loghandlers.DeferredRotatingFileHandler',
            'level': 'DEBUG',
            'formatter': 'verbose',
            'filename': '/dev/null',
            'maxBytes': 50000000,  # 50 Meg bytes
            'backupCount': 5,
            },
        },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
            },
        'inventory': {
            'handlers': ('inventory_file', 'console', 'mail_admins',),
            'level': 'ERROR',
            'propagate': True,
            },
        'api': {
            'handlers': ('api_file', 'console', 'mail_admins',),
            'level': 'ERROR',
            'propagate': True,
            },
        }
    }
<commit_msg>Added a contect processor so STATIC_URLS is available in the template context.<commit_after># -*- coding: utf-8 -*-
#
# base.py
#
# Django settings for inventory project.
#

import os
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

from inventory.apps.items.settings import *
from inventory.apps.login.settings import *

class IPList(list):

    def __init__(self, ips):
        try:
            #http://software.inl.fr/trac/wiki/IPy
            #ubuntu: apt-get install python-ipy
            from IPy import IP
            for ip in ips:
                self.append(IP(ip))
        except ImportError:
            pass

    def __contains__(self, ip):
        try:
            for net in self:
                if ip in net:
                    return True
        except:
            pass

        return False

DEBUG = False

ADMINS = (
    ('Sysadmins', 'carl.nobile@gmail.com'),
    )

ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

SITE_NAME = "TetraSys Inventory"

# Where is the 'website' directory with settings dir, apps, urls.py, etc. are.
SITE_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

# Where is the root of the site? This can be a root-relative URL.
SITE_URL = '/'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = os.path.join(SITE_ROOT, 'media/')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = os.path.join(SITE_URL, 'media/')

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = os.path.join(SITE_ROOT, 'static/')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = os.path.join(SITE_URL, 'static/')

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'dev/'),
    )

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    #'django.contrib.staticfiles.finders.DefaultStorageFinder',
    )

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'bjwykl6a7km26!0bsx%$v8g#s=+s5-(v2&d0^r8tl5++4ip$u4'

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend', # default
    'guardian.backends.ObjectPermissionBackend',
    )

# Guardian requirement
ANONYMOUS_USER_ID = -1

MIDDLEWARE_CLASSES = [
    # UpdateCacheMiddleware must be first on the list
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware', # This must be last
    ]

CACHES = {
    'default': {
        'BACKEND': 'redis_cache.RedisCache',
        'LOCATION': '/var/run/redis/redis.sock',
        },
    }

SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

ROOT_URLCONF = 'inventory.urls'

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'inventory.wsgi.application'

AUTH_USER_MODEL = 'accounts.User'

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.admindocs',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'dcolumn.dcolumns',
    'rest_framework',
    'oauth2_provider',
    #'guardian',
    'inventory.common',
    'inventory.accounts',
    'inventory.oauth2',
    'inventory.projects',
    'inventory.regions',
    'inventory.suppliers',
    'inventory.categories',
    'inventory.maintenance',
    #'inventory.objects',
    # Will be removed over time.
    'inventory.apps.items',
    'inventory.apps.login',
    'inventory.apps.reports',
    ]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(SITE_ROOT, 'templates'),
            ],
#        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'django.core.context_processors.static',
                ],
            'debug': DEBUG,
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
                #'django.template.loaders.eggs.Loader',
                ],
        },
    },
]

OAUTH2_PROVIDER = {
    # This is the list of available scopes.
    'SCOPES': {
        'read': 'Read scope',
        'write': 'Write scope',
        'groups': 'Access to your groups'
        }
    }

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
        ),
    'DEFAULT_PARSER_CLASSES': (
        'rest_framework.parsers.JSONParser',
        'rest_framework_xml.parsers.XMLParser',
        'rest_framework_yaml.parsers.YAMLParser',
        ),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.BrowsableAPIRenderer',
        'rest_framework.renderers.JSONRenderer',
        'rest_framework_xml.renderers.XMLRenderer',
        'rest_framework_yaml.renderers.YAMLRenderer',
        ),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework.authentication.SessionAuthentication',
        'oauth2_provider.ext.rest_framework.OAuth2Authentication',
        ),
    'TEST_REQUEST_DEFAULT_FORMAT': 'json',
    'TEST_REQUEST_RENDERER_CLASSES': (
        'rest_framework.renderers.MultiPartRenderer',
        'rest_framework.renderers.JSONRenderer',
        'rest_framework.renderers.TemplateHTMLRenderer'
        )
    }

# A sample logging configuration. The only tangible logging performed by this
# configuration is to send an email to the site admins on every HTTP 500 error
# when DEBUG=False. See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOG_DIR = os.path.abspath(os.path.join(SITE_ROOT, '..', 'logs'))
not os.path.isdir(LOG_DIR) and os.mkdir(LOG_DIR, 0775)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': "%(asctime)s %(levelname)s %(module)s %(funcName)s "
            "[line:%(lineno)d] %(message)s"
            },
        'simple': {
            'format': '%(asctime)s %(levelname)s %(message)s'
            },
        },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
            },
        },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
            },
        'console': {
            'level': 'WARNING',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
            },
        # Remove this one in the future.
        'inventory_file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'DEBUG',
            'formatter': 'verbose',
            'filename': '/dev/null',
            'maxBytes': 50000000,  # 50 Meg bytes
            'backupCount': 5,
            },
        'api_file': {
            'class': 'inventory.common.loghandlers.DeferredRotatingFileHandler',
            'level': 'DEBUG',
            'formatter': 'verbose',
            'filename': '/dev/null',
            'maxBytes': 50000000,  # 50 Meg bytes
            'backupCount': 5,
            },
        },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
            },
        'inventory': {
            'handlers': ('inventory_file', 'console', 'mail_admins',),
            'level': 'ERROR',
            'propagate': True,
            },
        'api': {
            'handlers': ('api_file', 'console', 'mail_admins',),
            'level': 'ERROR',
            'propagate': True,
            },
        }
    }
<|endoftext|>


<commit_before>from pywps import Process
from pywps import LiteralInput, LiteralOutput
from pywps import BoundingBoxInput, BoundingBoxOutput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS

import logging
LOGGER = logging.getLogger("PYWPS")

class InOut(Process):
    """
    This process defines several types of literal type of in- and outputs.

    TODO: add literal input with value range[(0,100)] ... see pywps doc
    """

    def __init__(self):
        inputs = [
            LiteralInput('string', 'String', data_type='string',
                         default="This is just a string"),
            LiteralInput('int', 'Integer', data_type='integer',
                         default="7"),
            LiteralInput('float', 'Float', data_type='float',
                         default="3.14"),
            LiteralInput('boolean', 'Boolean', data_type='boolean',
                         default='True'),
            LiteralInput('time', 'Time', data_type='time'),
            LiteralInput('string_choice', 'String Choice', data_type='string',
                         allowed_values=['one', 'two', 'three'],
                         default='two'),
            LiteralInput('string_multiple_choice', 'String Multiple Choice', data_type='string',
                         allowed_values=['one', 'two', 'three', 'four'],
                         min_occurs=1, max_occurs=2,
                         default='three'),
            # BoundingBoxInput('bbox', 'Bounding Box', ['epsg:4326', 'epsg:3035']),
            ComplexInput('text', 'Text',
                         supported_formats=[Format('text/plain')])

        ]
        outputs = [
            LiteralOutput('string', 'String', data_type='string'),
            LiteralOutput('int', 'Integer', data_type='integer'),
            LiteralOutput('float', 'Float', data_type='float'),
            LiteralOutput('boolean', 'Boolean', data_type='boolean'),
            LiteralOutput('time', 'Time', data_type='time'),
            LiteralOutput('string_choice', 'String Choice',
                          data_type='string'),
            LiteralOutput('string_multiple_choice', 'String Multiple Choice',
                          data_type='string'),
            # BoundingBoxOutput('bbox', 'Boudning Box', ['epsg:4326']),
            ComplexOutput('text', 'Text',
                          as_reference=True,
                          supported_formats=[Format('text/plain')]),
        ]

        super(InOut, self).__init__(
            self._handler,
            identifier="inout",
            title="InOut",
            version="1.0",
            abstract="Testing all WPS input and output parameters.",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True
            )

    def _handler(self, request, response):
        response.outputs['string'].data = request.inputs['string'][0].data
        response.outputs['int'].data = request.inputs['int'][0].data
        response.outputs['float'].data = request.inputs['float'][0].data
        response.outputs['boolean'].data = request.inputs['boolean'][0].data
        response.outputs['time'].data = request.inputs['time'][0].data
        response.outputs['string_choice'].data = \
            request.inputs['string_choice'][0].data
        response.outputs['string_multiple_choice'].data = ', '.join(
            [inpt.data for inpt in request.inputs['string_multiple_choice']])
        # TODO: bbox is not working
        # response.outputs['bbox'].data = request.inputs['bbox'][0].data
        response.outputs['text'].output_format = FORMATS.TEXT
        # TODO: how to copy file?
        response.outputs['text'].file = request.inputs['text'][0].file
        return response
<commit_msg>added abstracts<commit_after>from pywps import Process
from pywps import LiteralInput, LiteralOutput
from pywps import BoundingBoxInput, BoundingBoxOutput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS

import logging
LOGGER = logging.getLogger("PYWPS")

class InOut(Process):
    """
    This process defines several types of literal type of in- and outputs.

    TODO: add literal input with value range[(0,100)] ... see pywps doc
    """

    def __init__(self):
        inputs = [
            LiteralInput('string', 'String', data_type='string',
                         abstract='Enter a simple string.',
                         default="This is just a string"),
            LiteralInput('int', 'Integer', data_type='integer',
                         abstract='Enter an integer number.',
                         default="7"),
            LiteralInput('float', 'Float', data_type='float',
                         abstract='Enter a float number.',
                         default="3.14"),
            # TODO: boolean default is not displayed in phoenix
            LiteralInput('boolean', 'Boolean', data_type='boolean',
                         abstract='Make your choice :)',
                         default='1'),
            LiteralInput('time', 'Time', data_type='time',
                         abstract='Enter a time like 2016-09-02.',
                         default='2016-09-02'),
            LiteralInput('string_choice', 'String Choice', data_type='string',
                         abstract='Choose one item form list.',
                         allowed_values=['one', 'two', 'three'],
                         default='two'),
            LiteralInput('string_multiple_choice', 'String Multiple Choice',
                         abstract='Choose one or two items from list.',
                         data_type='string',
                         allowed_values=['one', 'two', 'three', 'four'],
                         min_occurs=1, max_occurs=2,
                         default='three'),
            # BoundingBoxInput('bbox', 'Bounding Box', ['epsg:4326', 'epsg:3035']),
            ComplexInput('text', 'Text',
                         abstract='Enter a URL pointing to a text document (optional)',
                         min_occurs=0,
                         supported_formats=[Format('text/plain')])

        ]
        outputs = [
            LiteralOutput('string', 'String', data_type='string'),
            LiteralOutput('int', 'Integer', data_type='integer'),
            LiteralOutput('float', 'Float', data_type='float'),
            LiteralOutput('boolean', 'Boolean', data_type='boolean'),
            LiteralOutput('time', 'Time', data_type='time'),
            LiteralOutput('string_choice', 'String Choice',
                          data_type='string'),
            LiteralOutput('string_multiple_choice', 'String Multiple Choice',
                          data_type='string'),
            # BoundingBoxOutput('bbox', 'Boudning Box', ['epsg:4326']),
            ComplexOutput('text', 'Text',
                          as_reference=True,
                          supported_formats=[Format('text/plain')]),
        ]

        super(InOut, self).__init__(
            self._handler,
            identifier="inout",
            title="InOut",
            version="1.0",
            abstract="Testing all WPS input and output parameters.",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True
            )

    def _handler(self, request, response):
        response.outputs['string'].data = request.inputs['string'][0].data
        response.outputs['int'].data = request.inputs['int'][0].data
        response.outputs['float'].data = request.inputs['float'][0].data
        response.outputs['boolean'].data = request.inputs['boolean'][0].data
        response.outputs['time'].data = request.inputs['time'][0].data
        response.outputs['string_choice'].data = \
            request.inputs['string_choice'][0].data
        response.outputs['string_multiple_choice'].data = ', '.join(
            [inpt.data for inpt in request.inputs['string_multiple_choice']])
        # TODO: bbox is not working
        # response.outputs['bbox'].data = request.inputs['bbox'][0].data
        # TODO: how to copy file?
        if 'text' in request.inputs:
            response.outputs['text'].output_format = FORMATS.TEXT
            response.outputs['text'].file = request.inputs['text'][0].file
        return response
<|endoftext|>