<commit_before>import json
import urllib.request

class Payee:

    def __init__(self, address):
        self.address = address

    def is_valid_address(self):
        """
        Does simple validation of a bitcoin-like address.
        Source: http://bit.ly/17OhFP5
        param : address : an ASCII or unicode string, of a bitcoin address.
        returns : boolean, indicating that the address has a correct format.
        """

        # The first character indicates the "version" of the address.
        chars_ok_first = "123"
        # alphanumeric characters without : l I O 0
        chars_ok = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

        # We do not check the high length limit of the address.
        # Usually, it is 35, but nobody knows what could happen in the future.
        if len(self.address) < 27:
            return False
        elif self.address[0] not in chars_ok_first:
            return False

        # We use the function "all" by passing it an enumerator as parameter.
        # It does a little optimization :
        # if one of the character is not valid, the next ones are not tested.
        return all((char in chars_ok for char in self.address[1:]))

    def get_addr_data(self):
        """
        Get the raw address data for the address from CounterPartyChain.
        :return: json, Raw data from CounterPartyChain.
        """
        api_url = 'https://counterpartychain.io/api/balances/'
        response = urllib.request.urlopen(api_url + self.address).read().decode("utf-8")
        json_response = json.loads(response)
        return json_response

    def get_balance(self):
        """
        Checks the balance of SJCX on the passed address from CounterPartyChain.
        :return: int, Number of SJCX in the address.
        """
        return int(self.get_addr_data()["data"][0]["amount"])<commit_msg>Fixed Tests<commit_after>import json
import urllib.request

class Payee:

    def __init__(self, address):
        self.address = address

    def is_valid_address(self):
        """
        Does simple validation of a bitcoin-like address.
        Source: http://bit.ly/17OhFP5
        param : address : an ASCII or unicode string, of a bitcoin address.
        returns : boolean, indicating that the address has a correct format.
        """

        # The first character indicates the "version" of the address.
        chars_ok_first = "123"
        # alphanumeric characters without : l I O 0
        chars_ok = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

        # We do not check the high length limit of the address.
        # Usually, it is 35, but nobody knows what could happen in the future.
        if len(self.address) < 27:
            return False
        elif self.address[0] not in chars_ok_first:
            return False

        # We use the function "all" by passing it an enumerator as parameter.
        # It does a little optimization :
        # if one of the character is not valid, the next ones are not tested.
        return all((char in chars_ok for char in self.address[1:]))

    def get_addr_data(self):
        """
        Get the raw address data for the address from CounterPartyChain.
        :return: json, Raw data from CounterPartyChain.
        """
        api_url = 'https://counterpartychain.io/api/balances/'
        response = urllib.request.urlopen(api_url + self.address).read().decode("utf-8")
        json_response = json.loads(response)
        return json_response

    def get_balance(self):
        """
        Checks the balance of SJCX on the passed address from CounterPartyChain.
        :return: int, Number of SJCX in the address.
        """
        data = self.get_addr_data()
        if data["success"] == 0:
            return 0
        for asset in data["data"]:
            if asset["asset"] == "SJCX":
                return int(float(asset["amount"]))
        return 0<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
VERSION = (0, 3, 152, 'final')

# pragma: no cover
if VERSION[-1] != "final":
    __version__ = '.'.join(map(str, VERSION))
else:
    # pragma: no cover
    __version__ = '.'.join(map(str, VERSION[:-1]))

default_app_config = 'djconnectwise.apps.DjangoConnectwiseConfig'
<commit_msg>rollback version no<commit_after># -*- coding: utf-8 -*-
VERSION = (0, 3, 151, 'final')

# pragma: no cover
if VERSION[-1] != "final":
    __version__ = '.'.join(map(str, VERSION))
else:
    # pragma: no cover
    __version__ = '.'.join(map(str, VERSION[:-1]))

default_app_config = 'djconnectwise.apps.DjangoConnectwiseConfig'
<|endoftext|>


<commit_before>"""payu.manifest
   ===============

   Provides an manifest class to store manifest data, which uses a
   subclassed yamanifest PayuManifest class

   :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""

# Python3 preparation
from __future__ import print_function, absolute_import

# Local
from payu import envmod
from payu.fsops import make_symlink, get_git_revision_hash

# External
from yamanifest.manifest import Manifest as YaManifest
import yamanifest as ym
from copy import deepcopy

import os, sys, fnmatch
import shutil
from distutils.dir_util import mkpath

# fast_hashes = ['nchash','binhash']
fast_hashes = ['binhash']
full_hashes = ['md5']
all_hashes = fast_hashes + full_hashes

class PayuManifest(YaManifest):
    """
    A manifest object sub-classed from yamanifest object with some payu specific
    additions and enhancements
    """

    def __init__(self, path, hashes=None, ignore=None, **kwargs):
        super(PayuManifest, self).__init__(path, hashes, **kwargs)

        if ignore is not None:
            self.ignore = ignore

    def check_fast(self, reproduce=False, **args):
        """
        Check hash value for all filepaths using a fast hash function and fall back to slower
        full hash functions if fast hashes fail to agree
        """
        hashvals = {}
        # Run a fast check
        if not self.check_file(filepaths=self.data.keys(),hashvals=hashvals,hashfn=fast_hashes,shortcircuit=True,**args):

            # Save all the fast hashes for failed files that we've already calculated
            for filepath in hashvals:
                for hash, val in hashvals[filepath].items():
                    self.data[filepath]["hashes"][hash] = val

            if reproduce:
                for filepath in hashvals:
                    print("Check failed for {} {}".format(filepath,hashvals[filepath]))
                    tmphash = {}
                    if self.check_file(filepaths=filepath,hashfn=full_hashes,hashvals=tmphash,shortcircuit=False,**args):
                        # File is still ok, so replace fast hashes
                        print("Full hashes ({}) checked ok".format(full_hashes))
                        print("Updating fast hashes for {} in {}".format(filepath,self.path))
                        self.add_fast(filepath,force=True)
                        print("Saving updated manifest")
                        self.dump()
                    else:
                        sys.stderr.write("Run cannot reproduce: manifest {} is not correct\n".format(self.path))
                        for path, hashdict in tmphash.items():
                            print("    {}:".format(path))
                            for hash, val in hashdict.items():
                                print("        {}: {} != {}".format(hash,val,self.data[path]['hashes'].get(hash,None)))
                        sys.exit(1)
            else:
                # Not relevant if full hashes are correct. Regenerate full hashes for all
                # filepaths that failed fast check
                print("Updating full hashes for {} files in {}".format(len(hashvals),self.path))

                # Add all full hashes at once -- much faster. Definitely want to force
                # the full hash to be updated. In the specific case of an empty hash the
                # value will be None, without force it will be written as null
                self.add(filepaths=list(hashvals.keys()),hashfn=full_hashes,force=True)

                # Write updates to version on disk
                print("Writing {}".format(self.path))
                self.dump()

    def dump(self):
        """
        Add git hash to header before dumping the file
        """
        self.header['githash'] = get_git_revision_hash()
        super(PayuManifest, self).dump()

    def add_filepath(self, filepath, fullpath, copy=False):
        """
        Bespoke function to add filepath & fullpath to manifest
        object without hashing. Can defer hashing until all files are
        added. Hashing all at once is much faster as overhead for
        threading is spread over all files
        """

        # Ignore directories
        if os.path.isdir(fullpath):
            return

        # Ignore anything matching the ignore patterns
        for pattern in self.ignore:
            if fnmatch.fnmatch(os.path.basename(fullpath), pattern):
                return

        if filepath not in self.data:
            self.data[filepath] = {}

        self.data[filepath]['fullpath'] = fullpath
        if 'hashes' not in self.data[filepath]:
            self.data[filepath]['hashes'] = {hash: None for hash in all_hashes}

        if copy:
            self.data[filepath]['copy'] = copy

    def add_fast(self, filepath, hashfn=fast_hashes, force=False):
        """
        Bespoke function to add filepaths but set shortcircuit to True, which means
        only the first calculatable hash will be stored. In this way only one "fast"
        hashing function need be called for each filepath
        """
        self.add(filepath, hashfn, force, shortcircuit=True)

    def copy_file(self, filepath):
        """
        Returns flag which says to copy rather than link a file
        """
        copy_file = False
        try:
            copy_file = self.data[filepath]['copy']
        except KeyError:
            return False
        return copy_file

    def make_links(self):
        """
        Payu integration function for creating symlinks in work directories which point
        back to the original file
        """
        for filepath in self:
            # print("Linking {}".format(filepath))
            # Don't try and link to itself, which happens when there is a real
            # file in the work directory, and not a symbolic link
            # if not os.path.realpath(filepath) == self.fullpath(filepath):
            #     make_symlink(self.fullpath(filepath), filepath)
            if self.copy_file(filepath):
                shutil.copy(self.fullpath(filepath), filepath)
            else:
                make_symlink(self.fullpath(filepath), filepath)

    def copy(self, path):
        """
        Copy myself to another location
        """
        shutil.copy(self.path, path)

class Manifest(object):
    """
    A Manifest class which stores all manifests for file tracking and
    methods to operate on them
    """

    def __init__(self, expt, reproduce):

        # Inherit experiment configuration
        self.expt = expt
        self.reproduce = reproduce

        # Manifest control configuration
        self.manifest_config = self.expt.config.get('manifest', {})

        self.have_exe_manifest = False
        self.have_input_manifest = False
        self.have_restart_manifest = False

        # If the run sets reproduce, default to reproduce executables. Allow user
        # to specify not to reproduce executables (might not be feasible if
        # executables don't match platform, or desirable if bugs existed in old exe)
        self.reproduce_exe = self.reproduce and self.manifest_config.get('reproduce_exe',True)

        # Not currently supporting specifying hash functions
        # self.hash_functions = manifest_config.get('hashfns', ['nchash','binhash','md5'])

        self.ignore = self.manifest_config.get('ignore',['.*'])
        self.ignore = [self.ignore] if isinstance(self.ignore, str) else self.ignore

        # Intialise manifests
        self.input_manifest = PayuManifest('manifests/input.yaml', ignore=self.ignore)
        self.restart_manifest = PayuManifest('manifests/restart.yaml', ignore=self.ignore)
        self.exe_manifest = PayuManifest('manifests/exe.yaml', ignore=self.ignore)

        # Make sure the manifests directory exists
        mkpath(os.path.dirname(self.exe_manifest.path))

    def setup(self):

        # Check if manifest files exist
        self.have_restart_manifest = os.path.exists(self.restart_manifest.path)

        if os.path.exists(self.input_manifest.path) and not self.manifest_config.get('overwrite',False):
            # Read manifest
            print("Loading input manifest: {}".format(self.input_manifest.path))
            self.input_manifest.load()

            if len(self.input_manifest) > 0:
                self.have_input_manifest = True

        if os.path.exists(self.exe_manifest.path):
            # Read manifest
            print("Loading exe manifest: {}".format(self.exe_manifest.path))
            self.exe_manifest.load()

            if len(self.exe_manifest) > 0:
                self.have_exe_manifest = True

        if self.reproduce:

            # Read restart manifest
            print("Loading restart manifest: {}".format(self.restart_manifest.path))
            self.restart_manifest.load()

            # MUST have input and restart manifests to be able to reproduce a run
            if len(self.restart_manifest) > 0:
                self.have_restart_manifest = True
            else:
                print("Restart manifest cannot be empty if reproduce is True")
                exit(1)

            if not self.have_input_manifest:
                print("Input manifest cannot be empty if reproduce is True")
                exit(1)

            if self.reproduce_exe and not self.have_exe_manifest:
                print("Executable manifest cannot empty if reproduce and reproduce_exe are True")
                exit(1)

            for model in self.expt.models:
                model.have_restart_manifest = True

            # Inspect the restart manifest for an appropriate value of # experiment
            # counter if not specified on the command line (and this env var set)
            if not os.environ.get('PAYU_CURRENT_RUN'):
                for filepath in self.restart_manifest:
                    head = os.path.dirname(self.restart_manifest.fullpath(filepath))
                    # Inspect each element of the fullpath looking for restartxxx style
                    # directories. Exit
                    while True:
                        head, tail = os.path.split(head)
                        if tail.startswith('restart'):
                            try:
                                n = int(tail.lstrip('restart'))
                            except ValueError:
                                pass
                            else:
                                self.expt.counter = n + 1
                                break

                    # Short circuit as soon as restart dir found
                    if self.expt.counter == 0: break

        else:

            self.have_restart_manifest = False

            # # Generate a restart manifest
            # for model in self.expt.models:
            #     if model.prior_restart_path is not None:
            #         # Try and find a manifest file in the restart dir
            #         restart_mf = PayuManifest.find_manifest(model.prior_restart_path)
            #         if restart_mf is not None:
            #             print("Loading restart manifest: {}".format(os.path.join(model.prior_restart_path,restart_mf.path)))
            #             self.restart_manifest.update(restart_mf,newpath=os.path.join(model.work_init_path_local))
            #             # Have two flags, one per model, the other controls if there is a call
            #             # to make_links in setup()
            #             model.have_restart_manifest = True
            #             # self.have_restart_manifest = True

    def make_links(self):

        print("Making links from manifests")
        self.exe_manifest.make_links()
        self.input_manifest.make_links()
        self.restart_manifest.make_links()

        print("Checking exe and input manifests")
        self.exe_manifest.check_fast(reproduce=self.reproduce_exe)
        self.input_manifest.check_fast(reproduce=self.reproduce)
        if self.reproduce:
            print("Checking restart manifest")
        else:
            print("Creating restart manifest")
        self.restart_manifest.check_fast(reproduce=self.reproduce)

    def copy_manifests(self, path):

        mkpath(path)
        try:
            self.exe_manifest.copy(path)
            self.input_manifest.copy(path)
            self.restart_manifest.copy(path)
        except IOError:
            pass<commit_msg>Delay writing of manifest files to allow multiple conditions to requite a sync to disk.<commit_after>"""payu.manifest
   ===============

   Provides an manifest class to store manifest data, which uses a
   subclassed yamanifest PayuManifest class

   :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""

# Python3 preparation
from __future__ import print_function, absolute_import

# Local
from payu import envmod
from payu.fsops import make_symlink, get_git_revision_hash

# External
from yamanifest.manifest import Manifest as YaManifest
import yamanifest as ym
from copy import deepcopy

import os, sys, fnmatch
import shutil
from distutils.dir_util import mkpath

# fast_hashes = ['nchash','binhash']
fast_hashes = ['binhash']
full_hashes = ['md5']
all_hashes = fast_hashes + full_hashes

class PayuManifest(YaManifest):
    """
    A manifest object sub-classed from yamanifest object with some payu specific
    additions and enhancements
    """

    def __init__(self, path, hashes=None, ignore=None, **kwargs):
        super(PayuManifest, self).__init__(path, hashes, **kwargs)

        if ignore is not None:
            self.ignore = ignore

        self.needsync = False

    def check_fast(self, reproduce=False, **args):
        """
        Check hash value for all filepaths using a fast hash function and fall back to slower
        full hash functions if fast hashes fail to agree
        """
        hashvals = {}
        # Run a fast check
        if not self.check_file(filepaths=self.data.keys(),hashvals=hashvals,hashfn=fast_hashes,shortcircuit=True,**args):

            # Save all the fast hashes for failed files that we've already calculated
            for filepath in hashvals:
                for hash, val in hashvals[filepath].items():
                    self.data[filepath]["hashes"][hash] = val

            if reproduce:
                for filepath in hashvals:
                    print("Check failed for {} {}".format(filepath,hashvals[filepath]))
                    tmphash = {}
                    if self.check_file(filepaths=filepath,hashfn=full_hashes,hashvals=tmphash,shortcircuit=False,**args):
                        # File is still ok, so replace fast hashes
                        print("Full hashes ({}) checked ok".format(full_hashes))
                        print("Updating fast hashes for {} in {}".format(filepath,self.path))
                        self.add_fast(filepath,force=True)
                        print("Saving updated manifest")
                        self.needsync = True
                    else:
                        sys.stderr.write("Run cannot reproduce: manifest {} is not correct\n".format(self.path))
                        for path, hashdict in tmphash.items():
                            print("    {}:".format(path))
                            for hash, val in hashdict.items():
                                print("        {}: {} != {}".format(hash,val,self.data[path]['hashes'].get(hash,None)))
                        sys.exit(1)
            else:
                # Not relevant if full hashes are correct. Regenerate full hashes for all
                # filepaths that failed fast check
                print("Updating full hashes for {} files in {}".format(len(hashvals),self.path))

                # Add all full hashes at once -- much faster. Definitely want to force
                # the full hash to be updated. In the specific case of an empty hash the
                # value will be None, without force it will be written as null
                self.add(filepaths=list(hashvals.keys()),hashfn=full_hashes,force=True)

                # Flag need to update version on disk
                self.needsync = True

    def dump(self):
        """
        Add git hash to header before dumping the file
        """
        self.header['githash'] = get_git_revision_hash()
        super(PayuManifest, self).dump()

    def add_filepath(self, filepath, fullpath, copy=False):
        """
        Bespoke function to add filepath & fullpath to manifest
        object without hashing. Can defer hashing until all files are
        added. Hashing all at once is much faster as overhead for
        threading is spread over all files
        """

        # Ignore directories
        if os.path.isdir(fullpath):
            return

        # Ignore anything matching the ignore patterns
        for pattern in self.ignore:
            if fnmatch.fnmatch(os.path.basename(fullpath), pattern):
                return

        if filepath not in self.data:
            self.data[filepath] = {}

        self.data[filepath]['fullpath'] = fullpath
        if 'hashes' not in self.data[filepath]:
            self.data[filepath]['hashes'] = {hash: None for hash in all_hashes}

        if copy:
            self.data[filepath]['copy'] = copy

    def add_fast(self, filepath, hashfn=fast_hashes, force=False):
        """
        Bespoke function to add filepaths but set shortcircuit to True, which means
        only the first calculatable hash will be stored. In this way only one "fast"
        hashing function need be called for each filepath
        """
        self.add(filepath, hashfn, force, shortcircuit=True)

    def copy_file(self, filepath):
        """
        Returns flag which says to copy rather than link a file
        """
        copy_file = False
        try:
            copy_file = self.data[filepath]['copy']
        except KeyError:
            return False
        return copy_file

    def make_links(self):
        """
        Payu integration function for creating symlinks in work directories which point
        back to the original file
        """
        delete_list = []
        for filepath in self:
            # Check file exists. It may have been deleted but still in manifest
            if not os.path.exists(self.fullpath(filepath)):
                delete_list.append(filepath)
                continue

            if self.copy_file(filepath):
                shutil.copy(self.fullpath(filepath), filepath)
            else:
                make_symlink(self.fullpath(filepath), filepath)

        for filepath in delete_list:
            print("File not found: {} removing from manifest".format(self.fullpath(filepath)))
            self.delete(filepath)
            self.needsync = True

    def copy(self, path):
        """
        Copy myself to another location
        """
        shutil.copy(self.path, path)

class Manifest(object):
    """
    A Manifest class which stores all manifests for file tracking and
    methods to operate on them
    """

    def __init__(self, expt, reproduce):

        # Inherit experiment configuration
        self.expt = expt
        self.reproduce = reproduce

        # Manifest control configuration
        self.manifest_config = self.expt.config.get('manifest', {})

        self.have_exe_manifest = False
        self.have_input_manifest = False
        self.have_restart_manifest = False

        # If the run sets reproduce, default to reproduce executables. Allow user
        # to specify not to reproduce executables (might not be feasible if
        # executables don't match platform, or desirable if bugs existed in old exe)
        self.reproduce_exe = self.reproduce and self.manifest_config.get('reproduce_exe',True)

        # Not currently supporting specifying hash functions
        # self.hash_functions = manifest_config.get('hashfns', ['nchash','binhash','md5'])

        self.ignore = self.manifest_config.get('ignore',['.*'])
        self.ignore = [self.ignore] if isinstance(self.ignore, str) else self.ignore

        # Intialise manifests
        self.input_manifest = PayuManifest('manifests/input.yaml', ignore=self.ignore)
        self.restart_manifest = PayuManifest('manifests/restart.yaml', ignore=self.ignore)
        self.exe_manifest = PayuManifest('manifests/exe.yaml', ignore=self.ignore)

        # Make sure the manifests directory exists
        mkpath(os.path.dirname(self.exe_manifest.path))

    def setup(self):

        # Check if manifest files exist
        self.have_restart_manifest = os.path.exists(self.restart_manifest.path)

        if os.path.exists(self.input_manifest.path) and not self.manifest_config.get('overwrite',False):
            # Read manifest
            print("Loading input manifest: {}".format(self.input_manifest.path))
            self.input_manifest.load()

            if len(self.input_manifest) > 0:
                self.have_input_manifest = True

        if os.path.exists(self.exe_manifest.path):
            # Read manifest
            print("Loading exe manifest: {}".format(self.exe_manifest.path))
            self.exe_manifest.load()

            if len(self.exe_manifest) > 0:
                self.have_exe_manifest = True

        if self.reproduce:

            # Read restart manifest
            print("Loading restart manifest: {}".format(self.restart_manifest.path))
            self.restart_manifest.load()

            # MUST have input and restart manifests to be able to reproduce a run
            if len(self.restart_manifest) > 0:
                self.have_restart_manifest = True
            else:
                print("Restart manifest cannot be empty if reproduce is True")
                exit(1)

            if not self.have_input_manifest:
                print("Input manifest cannot be empty if reproduce is True")
                exit(1)

            if self.reproduce_exe and not self.have_exe_manifest:
                print("Executable manifest cannot empty if reproduce and reproduce_exe are True")
                exit(1)

            for model in self.expt.models:
                model.have_restart_manifest = True

            # Inspect the restart manifest for an appropriate value of # experiment
            # counter if not specified on the command line (and this env var set)
            if not os.environ.get('PAYU_CURRENT_RUN'):
                for filepath in self.restart_manifest:
                    head = os.path.dirname(self.restart_manifest.fullpath(filepath))
                    # Inspect each element of the fullpath looking for restartxxx style
                    # directories. Exit
                    while True:
                        head, tail = os.path.split(head)
                        if tail.startswith('restart'):
                            try:
                                n = int(tail.lstrip('restart'))
                            except ValueError:
                                pass
                            else:
                                self.expt.counter = n + 1
                                break

                    # Short circuit as soon as restart dir found
                    if self.expt.counter == 0: break

        else:

            self.have_restart_manifest = False

            # # Generate a restart manifest
            # for model in self.expt.models:
            #     if model.prior_restart_path is not None:
            #         # Try and find a manifest file in the restart dir
            #         restart_mf = PayuManifest.find_manifest(model.prior_restart_path)
            #         if restart_mf is not None:
            #             print("Loading restart manifest: {}".format(os.path.join(model.prior_restart_path,restart_mf.path)))
            #             self.restart_manifest.update(restart_mf,newpath=os.path.join(model.work_init_path_local))
            #             # Have two flags, one per model, the other controls if there is a call
            #             # to make_links in setup()
            #             model.have_restart_manifest = True
            #             # self.have_restart_manifest = True

    def make_links(self):

        print("Making links from manifests")
        self.exe_manifest.make_links()
        self.input_manifest.make_links()
        self.restart_manifest.make_links()

        print("Checking exe and input manifests")
        self.exe_manifest.check_fast(reproduce=self.reproduce_exe)
        self.input_manifest.check_fast(reproduce=self.reproduce)

        if self.reproduce:
            print("Checking restart manifest")
        else:
            print("Creating restart manifest")
        self.restart_manifest.check_fast(reproduce=self.reproduce)

        # Write updates to version on disk
        if self.exe_manifest.needsync:
            print("Writing {}".format(self.exe_manifest.path))
            self.exe_manifest.dump()
        if self.input_manifest.needsync:
            print("Writing {}".format(self.input_manifest.path))
            self.input_manifest.dump()
        if self.restart_manifest.needsync:
            print("Writing {}".format(self.restart_manifest.path))
            self.restart_manifest.dump()

    def copy_manifests(self, path):

        mkpath(path)
        try:
            self.exe_manifest.copy(path)
            self.input_manifest.copy(path)
            self.restart_manifest.copy(path)
        except IOError:
            pass<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys
# env
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djequis.settings')

# required if using django models
import django
django.setup()

from django.db import connections

import argparse
import logging

logger = logging.getLogger('djequis')

'''
Shell script...
'''

# set up command-line options
desc = """
Accepts as input...
"""

parser = argparse.ArgumentParser(description=desc)

parser.add_argument(
    '-d', '--database',
    required=True,
    help="Database connection name from settings e.g. default",
    dest='database'
)
parser.add_argument(
    '-s', '--sql',
    required=True,
    help="A simple SQL incantation for testing e.g. 'SELECT * FROM auth_user'",
    dest='sql'
)
parser.add_argument(
    '--test',
    action='store_true',
    help="Dry run?",
    dest='test'
)

def main():
    '''
    main function
    '''

    cursor = connections[database].cursor()

    if test:
        print(sql)
    else:
        cursor.execute(sql)
        objects = cursor.fetchall()

        for o in objects:
            print(o)

######################
# shell command line
######################

if __name__ == '__main__':
    args = parser.parse_args()
    database = args.database
    sql = args.sql
    test = args.test

    if test:
        print(args)

    sys.exit(main())

<commit_msg>cleanup of repository<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys
# env
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djequis.settings')

# required if using django models
import django
django.setup()

from django.db import connections

import argparse
import logging

logger = logging.getLogger('djequis')

'''
Shell script...
'''

# set up command-line options
desc = """
Accepts as input...
"""

parser = argparse.ArgumentParser(description=desc)

parser.add_argument(
    '-d', '--database',
    required=True,
    help="Database connection name from settings e.g. default",
    dest='database'
)
parser.add_argument(
    '-s', '--sql',
    required=True,
    help="A simple SQL incantation for testing e.g. 'SELECT * FROM auth_user'",
    dest='sql'
)
parser.add_argument(
    '--test',
    action='store_true',
    help="Dry run?",
    dest='test'
)

def main():
    '''
    main function
    '''

    print("hello")

    cursor = connections[database].cursor()

    if test:
        print(sql)
    else:
        cursor.execute(sql)
        objects = cursor.fetchall()

        for o in objects:
            print(o)

######################
# shell command line
######################

if __name__ == '__main__':
    args = parser.parse_args()
    database = args.database
    sql = args.sql
    test = args.test

    if test:
        print(args)

    sys.exit(main())

<|endoftext|>


<commit_before>from Queue import Queue
import thread
import threading

from . import wrappers as w
from .wrappers import PcapError, PcapWarning

__all__ = [PcapError, PcapWarning, 'Pcap']
_pcap_user_mapping = dict()

@w.ffi.callback('void(u_char *user, const struct pcap_pkthdr *h, const u_char *bytes)')
def _packet_handler(user, hdr, buff):
    user = w.ffi.string(user)
    buff = w.ffi.buffer(buff, hdr.len)[:]

    obj = _pcap_user_mapping[user]
    obj._packet_handler(hdr, buff)

class _DefaultDecoder(object):
    def decoder(self, data):
        return data

class Pcap(object):
    def __init__(self, promisc=True, buffer_size=None, read_timeout=100, show_packets=False):
        self._pcap_t = None
        self._packets = None
        self._promisc = promisc
        self._buffer_size = buffer_size
        self._read_timeout = read_timeout
        self._show_packets = show_packets
        self._pcap_lock = threading.Lock()

        try:
            from impacket.ImpactDecoder import LinuxSLLDecoder
            self._decoder = LinuxSLLDecoder()
        except ImportError:
            self._decoder = None

    def _set_buffer(self, buffer_size):
        w.pcap_set_buffer_size(self._pcap_t, buffer_size)

    def _set_timeout(self, read_timeout):
        w.pcap_set_timeout(self._pcap_t, read_timeout)

    @property
    def packets(self):
        for p in iter(self._packets.get, None):
            yield p

    @property
    def packet_count(self):
        return self._packets.qsize()

    @property
    def activated(self):
        return self._pcap_t is not None

    @property
    def snaplen(self):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return w.pcap_snapshot(self._pcap_t)

    @snaplen.setter
    def snaplen(self, snaplen):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return w.pcap_set_snaplen(self._pcap_t, snaplen) == 0

    @property
    def promisc(self):
        return self._promisc

    def _set_promisc(self, state):
        w.pcap_set_promisc(self._pcap_t, state)

    @property
    def tstamp_types(self):
        return [(val, w.pcap_tstamp_type_val_to_name(val), w.pcap_tstamp_type_val_to_description(val)) for val in w.pcap_list_tstamp_types(self._pcap_t)]

    @property
    def datalink(self):
        dl = w.pcap_datalink(self._pcap_t)

        for dl_type, dl_name, dl_description in self.datalinks:
            if dl == dl_type:
                return dl_name

        return None

    def set_datalink(self, dlt):
        return w.pcap_set_datalink(self._pcap_t, dlt) == 0

    @property
    def datalinks(self):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return [(val, w.pcap_datalink_val_to_name(val), w.pcap_datalink_val_to_description(val)) for val in w.pcap_list_datalinks(self._pcap_t)]

    def set_tstamp_type(self, tstamp_type):
        return w.pcap_set_tstamp_type(self._pcap_t, tstamp_type) == 0

    def _activate(self):
        self._packets = Queue()
        w.pcap_activate(self._pcap_t)

    def _start(self):
        self._activate()
        self._start_loop()

    def _start_loop(self):
        thread.start_new_thread(self._loop, tuple())

    def _loop(self):
        user = str(id(self))
        _pcap_user_mapping[user] = self

        with self._pcap_lock:
            while self.activated:
                ret = w.pcap_dispatch(self._pcap_t, -1, _packet_handler, user)
                if ret is None:
                    break

        del _pcap_user_mapping[user]

    def close(self):
        with self._pcap_lock:
            if self._pcap_t:
                w.pcap_close(self._pcap_t)
                self._pcap_t = None

    def _packet_handler(self, hdr, packet):
        packet_dict = dict(
            tv_sec=hdr.ts.tv_sec,
            tv_usec=hdr.ts.tv_usec,
            len=hdr.len,
            caplen=hdr.caplen,
            data=packet,
            decoded=None
        )

        if self._decoder:
            packet_dict['decoded'] = self._decoder.decode(packet)

            if self._show_packets:
                print packet_dict['decoded']

        self._packets.put(packet_dict)

    def open_dev(self, dev='any'):
        """Create and activate pcap on a device. If no `dev` is specified then
        'any' i used.
        """
        self._pcap_t = w.pcap_create(dev)
        self._set_promisc(self._promisc)

        if self._buffer_size:
            self._set_buffer(self._buffer_size)

        if self._read_timeout:
            self._set_timeout(self._read_timeout)

        self._start()

    def open_file(self, filename):
        """Create and activate pcap with a file or filename."""

        if isinstance(filename, basestring):
            self._pcap_t = w.pcap_open_offline(filename)
        else:
            self._pcap_t = w.pcap_fopen_offline(filename)
        self._start()
<commit_msg>Remove unused decoder.<commit_after>from Queue import Queue
import thread
import threading

from . import wrappers as w
from .wrappers import PcapError, PcapWarning

__all__ = [PcapError, PcapWarning, 'Pcap']
_pcap_user_mapping = dict()

@w.ffi.callback('void(u_char *user, const struct pcap_pkthdr *h, const u_char *bytes)')
def _packet_handler(user, hdr, buff):
    user = w.ffi.string(user)
    buff = w.ffi.buffer(buff, hdr.len)[:]

    obj = _pcap_user_mapping[user]
    obj._packet_handler(hdr, buff)

class Pcap(object):
    def __init__(self, promisc=True, buffer_size=None, read_timeout=100, show_packets=False):
        self._pcap_t = None
        self._packets = None
        self._promisc = promisc
        self._buffer_size = buffer_size
        self._read_timeout = read_timeout
        self._show_packets = show_packets
        self._pcap_lock = threading.Lock()

        try:
            from impacket.ImpactDecoder import LinuxSLLDecoder
            self._decoder = LinuxSLLDecoder()
        except ImportError:
            self._decoder = None

    def _set_buffer(self, buffer_size):
        w.pcap_set_buffer_size(self._pcap_t, buffer_size)

    def _set_timeout(self, read_timeout):
        w.pcap_set_timeout(self._pcap_t, read_timeout)

    @property
    def packets(self):
        for p in iter(self._packets.get, None):
            yield p

    @property
    def packet_count(self):
        return self._packets.qsize()

    @property
    def activated(self):
        return self._pcap_t is not None

    @property
    def snaplen(self):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return w.pcap_snapshot(self._pcap_t)

    @snaplen.setter
    def snaplen(self, snaplen):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return w.pcap_set_snaplen(self._pcap_t, snaplen) == 0

    @property
    def promisc(self):
        return self._promisc

    def _set_promisc(self, state):
        w.pcap_set_promisc(self._pcap_t, state)

    @property
    def tstamp_types(self):
        return [(val, w.pcap_tstamp_type_val_to_name(val), w.pcap_tstamp_type_val_to_description(val)) for val in w.pcap_list_tstamp_types(self._pcap_t)]

    @property
    def datalink(self):
        dl = w.pcap_datalink(self._pcap_t)

        for dl_type, dl_name, dl_description in self.datalinks:
            if dl == dl_type:
                return dl_name

        return None

    def set_datalink(self, dlt):
        return w.pcap_set_datalink(self._pcap_t, dlt) == 0

    @property
    def datalinks(self):
        if not self.activated:
            raise PcapError(w.PCAP_ERROR_NOT_ACTIVATED)
        return [(val, w.pcap_datalink_val_to_name(val), w.pcap_datalink_val_to_description(val)) for val in w.pcap_list_datalinks(self._pcap_t)]

    def set_tstamp_type(self, tstamp_type):
        return w.pcap_set_tstamp_type(self._pcap_t, tstamp_type) == 0

    def _activate(self):
        self._packets = Queue()
        w.pcap_activate(self._pcap_t)

    def _start(self):
        self._activate()
        self._start_loop()

    def _start_loop(self):
        thread.start_new_thread(self._loop, tuple())

    def _loop(self):
        user = str(id(self))
        _pcap_user_mapping[user] = self

        with self._pcap_lock:
            while self.activated:
                ret = w.pcap_dispatch(self._pcap_t, -1, _packet_handler, user)
                if ret is None:
                    break

        del _pcap_user_mapping[user]

    def close(self):
        with self._pcap_lock:
            if self._pcap_t:
                w.pcap_close(self._pcap_t)
                self._pcap_t = None

    def _packet_handler(self, hdr, packet):
        packet_dict = dict(
            tv_sec=hdr.ts.tv_sec,
            tv_usec=hdr.ts.tv_usec,
            len=hdr.len,
            caplen=hdr.caplen,
            data=packet,
            decoded=None
        )

        if self._decoder:
            packet_dict['decoded'] = self._decoder.decode(packet)

            if self._show_packets:
                print packet_dict['decoded']

        self._packets.put(packet_dict)

    def open_dev(self, dev='any'):
        """Create and activate pcap on a device. If no `dev` is specified then
        'any' i used.
        """
        self._pcap_t = w.pcap_create(dev)
        self._set_promisc(self._promisc)

        if self._buffer_size:
            self._set_buffer(self._buffer_size)

        if self._read_timeout:
            self._set_timeout(self._read_timeout)

        self._start()

    def open_file(self, filename):
        """Create and activate pcap with a file or filename."""

        if isinstance(filename, basestring):
            self._pcap_t = w.pcap_open_offline(filename)
        else:
            self._pcap_t = w.pcap_fopen_offline(filename)
        self._start()
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

import paramiko
from time import sleep
import socket
import os

from toolz import merge

# These are handy for creating colorful terminal output to enhance readability
# of the output generated by dcluster.
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def start_center(logdir, center_addr, center_port):

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(center_addr, timeout = 20)

    cmd = 'dcenter --host {addr} --port {port}'.format(
        addr = center_addr, port = center_port, logdir = logdir)

    # Optionally re-direct stdout and stderr to a logfile
    if logdir is not None:
        cmd = 'mkdir -p {logdir} && ' + cmd
        cmd += '&> {logdir}/dcenter_{addr}:{port}.log\n'.format(
            addr = center_addr, port = center_port, logdir = logdir)

    # Run the command
    stdin, stdout, stderr = ssh.exec_command(cmd, get_pty=True)
    return {'address': center_addr, 'port': center_port, 'client': ssh, 'stdout': stdout, 'stderr': stderr}

def start_worker(logdir, center_addr, center_port, worker_addr, workers_per_node, cpus_per_worker):

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(worker_addr, timeout = 20)

    # Pick a random port for the worker.  This prevents contention over a single port,
    # which may occasionally cause workers to fail to register with the center node.
    import random
    worker_port = random.randint(10000, 20000)

    cmd = 'dworker {center_addr}:{center_port} --host {worker_addr} --port {worker_port}'.format(
        center_addr = center_addr, center_port = center_port,
        worker_addr = worker_addr, worker_port = worker_port,
        logdir = logdir)

    # (Optionally) add the ncores argument
    if cpus_per_worker is not None:
        cmd += ' --ncores {ncpu}\n'.format(ncpu = cpus_per_worker)

    # Optionally redirect stdout and stderr to a logfile
    if logdir is not None:
        cmd = 'mkdir -p {logdir} && ' + cmd
        cmd += '&> {logdir}/dcenter_{addr}:{port}.log\n'.format(
            addr = worker_addr, port = worker_port, logdir = logdir)

    # Run the command
    stdin, stdout, stderr = ssh.exec_command(cmd, get_pty=True)
    return {'address': worker_addr, 'port': worker_port, 'client': ssh, 'stdout': stdout, 'stderr': stderr}

class Cluster(object):
    def __init__(self, center_addr, center_port, worker_addrs, workers_per_node = 1, cpus_per_worker = None, logdir = None):

        self.center_addr = center_addr
        self.center_port = center_port
        self.workers_per_node = workers_per_node
        self.cpus_per_worker = cpus_per_worker

        # Generate a universal timestamp to use for log files
        import datetime
        if logdir is not None:
            logdir = os.path.join(logdir, "dcluster_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
            print(bcolors.WARNING + 'Output will be redirected to logfiles stored locally on individual woker nodes under "{logdir}".'.format(logdir=logdir) + bcolors.ENDC)
        self.logdir = logdir

        # Start the center node
        self.center = merge(start_center(logdir, center_addr, center_port), {'address': center_addr})

        # Start worker nodes
        self.workers = []
        for addr in worker_addrs:
            self.add_worker(addr)

    def monitor_remote_processes(self):

        # Set up channel timeouts (which we rely on below to make readline()
        # non-blocking. Also set up some nicely formatted output labels we can
        # prepend to each line of output, and create a 'status' key to keep
        # track of jobs that terminate prematurely.
        self.center['label'] = (bcolors.BOLD +
                                'center {addr}:{port}'.format(addr = self.center['address'],
                                                               port = self.center['port']) +
                                bcolors.ENDC)
        self.center['status'] = 'running'
        self.center['stdout'].channel.settimeout(0.1)
        self.center['stderr'].channel.settimeout(0.1)

        for worker in self.workers:
            worker['label'] = 'worker {addr}:{port} {{{worker_id}}}'.format(worker_id = worker['worker_id'],
                                                                            addr = worker['address'],
                                                                            port = worker['port'])
            worker['status'] = 'running'
            worker['stdout'].channel.settimeout(0.1)
            worker['stderr'].channel.settimeout(0.1)

        # Form a list containing all processes, since we treat them equally from here on out.
        all_processes = [self.center] + self.workers

        try:
            while(1):
                for process in all_processes:

                    if process['status'] == 'finished':
                        continue

                    if process['stdout'].channel.exit_status_ready():
                        exit_status = process['stdout'].channel.recv_exit_status()
                        print('[ ' + process['label'] + ' ] : ' + bcolors.FAIL +
                              "remote process exited with exit status " + str(exit_status) + bcolors.ENDC)
                        process['status'] = 'finished'

                    # Read stdout stream, time out if necessary.
                    try:
                        line = process['stdout'].readline()
                        while len(line) > 0:    # Loops until a timout exception occurs
                            print('[ {label} ] : {output}'.format(label = process['label'],
                                                                  output = line.rstrip()))
                            line = process['stdout'].readline()

                    except paramiko.buffered_pipe.PipeTimeout:
                        continue
                    except socket.timeout:
                        continue

                    # Read stderr stream, time out if necessary
                    try:
                        line = process['stderr'].readline()
                        while len(line) > 0:
                            print('[ {label} ] : ' +
                                  bcolors.WARNING +
                                  ' {output}'.format(label = process['label'],
                                                     output = line.rstrip()) +
                                  bcolors.ENDC)
                            line = process['stderr'].readline()

                    except paramiko.buffered_pipe.PipeTimeout:
                        continue
                    except socket.timeout:
                        continue

                # Kill some time and free up CPU before starting the next sweep
                # through the processes.
                sleep(1)

                # end while()

        except KeyboardInterrupt:
            pass   # Return execution to the calling process

    def add_worker(self, address):
        self.workers += [ merge(start_worker(self.logdir, self.center_addr, self.center_port, address,
                                             self.workers_per_node, self.cpus_per_worker),
                                {'address': address, 'worker_id': worker_id})
                          for worker_id in range(self.workers_per_node)]

    def shutdown(self):

        # Close down sockets
        for d in self.workers:
            d['stdout'].channel.close()
            d['stderr'].channel.close()

        self.center['stdout'].channel.close()
        self.center['stderr'].channel.close()

        # Close down ssh connections
        for d in self.workers:
            d['client'].close()  # Calling this multiple times on the same connection seems ok.

        self.center['client'].close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
<commit_msg>Switched back to using the invoke_shell() method to establish the ssh connection, as this seems to ensure that commands are run in a shell environment most similar to the interactive shell a user would use if they logged into a remote server.<commit_after>from __future__ import print_function, division, absolute_import

import paramiko
from time import sleep
import socket
import os

from toolz import merge

# These are handy for creating colorful terminal output to enhance readability
# of the output generated by dcluster.
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def start_center(logdir, center_addr, center_port):

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(center_addr, timeout = 20)
    channel = ssh.invoke_shell()

    cmd = 'dcenter --host {addr} --port {port}'.format(
        addr = center_addr, port = center_port, logdir = logdir)

    # Optionally re-direct stdout and stderr to a logfile
    if logdir is not None:
        cmd = 'mkdir -p {logdir} && ' + cmd
        cmd += '&> {logdir}/dcenter_{addr}:{port}.log\n'.format(
            addr = center_addr, port = center_port, logdir = logdir)

    # Run the command
    channel.send(cmd + '\n')

    # Return a dictionary including file handlers for stdout and stderr
    return {'address': center_addr, 'port': center_port, 'client': ssh,
            'stdout': channel.makefile('r'), 'stderr': channel.makefile_stderr('r')}

def start_worker(logdir, center_addr, center_port, worker_addr, workers_per_node, cpus_per_worker):

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(worker_addr, timeout = 20)
    channel = ssh.invoke_shell()

    # Pick a random port for the worker.  This prevents contention over a single port,
    # which may occasionally cause workers to fail to register with the center node.
    import random
    worker_port = random.randint(10000, 20000)

    cmd = 'dworker {center_addr}:{center_port} --host {worker_addr} --port {worker_port}'.format(
        center_addr = center_addr, center_port = center_port,
        worker_addr = worker_addr, worker_port = worker_port,
        logdir = logdir)

    # (Optionally) add the ncores argument
    if cpus_per_worker is not None:
        cmd += ' --ncores {ncpu}\n'.format(ncpu = cpus_per_worker)

    # Optionally redirect stdout and stderr to a logfile
    if logdir is not None:
        cmd = 'mkdir -p {logdir} && ' + cmd
        cmd += '&> {logdir}/dcenter_{addr}:{port}.log\n'.format(
            addr = worker_addr, port = worker_port, logdir = logdir)

    # Run the command
    channel.send(cmd + '\n')

    # Return a dictionary including file handlers for stdout and stderr
    return {'address': worker_addr, 'port': worker_port, 'client': ssh,
            'stdout': channel.makefile('r'), 'stderr': channel.makefile_stderr('r')}

class Cluster(object):
    def __init__(self, center_addr, center_port, worker_addrs, workers_per_node = 1, cpus_per_worker = None, logdir = None):

        self.center_addr = center_addr
        self.center_port = center_port
        self.workers_per_node = workers_per_node
        self.cpus_per_worker = cpus_per_worker

        # Generate a universal timestamp to use for log files
        import datetime
        if logdir is not None:
            logdir = os.path.join(logdir, "dcluster_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
            print(bcolors.WARNING + 'Output will be redirected to logfiles stored locally on individual woker nodes under "{logdir}".'.format(logdir=logdir) + bcolors.ENDC)
        self.logdir = logdir

        # Start the center node
        self.center = merge(start_center(logdir, center_addr, center_port), {'address': center_addr})

        # Start worker nodes
        self.workers = []
        for addr in worker_addrs:
            self.add_worker(addr)

    def monitor_remote_processes(self):

        # Set up channel timeouts (which we rely on below to make readline()
        # non-blocking. Also set up some nicely formatted output labels we can
        # prepend to each line of output, and create a 'status' key to keep
        # track of jobs that terminate prematurely.
        self.center['label'] = (bcolors.BOLD +
                                'center {addr}:{port}'.format(addr = self.center['address'],
                                                               port = self.center['port']) +
                                bcolors.ENDC)
        self.center['status'] = 'running'
        self.center['stdout'].channel.settimeout(0.1)
        self.center['stderr'].channel.settimeout(0.1)

        for worker in self.workers:
            worker['label'] = 'worker {addr}:{port} {{{worker_id}}}'.format(worker_id = worker['worker_id'],
                                                                            addr = worker['address'],
                                                                            port = worker['port'])
            worker['status'] = 'running'
            worker['stdout'].channel.settimeout(0.1)
            worker['stderr'].channel.settimeout(0.1)

        # Form a list containing all processes, since we treat them equally from here on out.
        all_processes = [self.center] + self.workers

        try:
            while(1):
                for process in all_processes:

                    if process['status'] == 'finished':
                        continue

                    # Read stdout stream, time out if necessary.
                    try:
                        line = process['stdout'].readline()
                        while len(line) > 0:    # Loops until a timout exception occurs
                            print('[ {label} ] : {output}'.format(label = process['label'],
                                                                  output = line.rstrip()))
                            line = process['stdout'].readline()

                    except paramiko.buffered_pipe.PipeTimeout:
                        continue
                    except socket.timeout:
                        continue

                    # Read stderr stream, time out if necessary
                    try:
                        line = process['stderr'].readline()
                        while len(line) > 0:
                            print('[ {{label}} ] : '.format(label = process['label']) +
                                  bcolors.FAIL + '{output}'.format(output = line.rstrip()) + bcolors.ENDC)
                            line = process['stderr'].readline()

                    except paramiko.buffered_pipe.PipeTimeout:
                        continue
                    except socket.timeout:
                        continue

                    # Check to see if the process has exited.
                    if process['stdout'].channel.exit_status_ready():
                        exit_status = process['stdout'].channel.recv_exit_status()
                        print('[ {{label}} ] : '.format(label = process['label']) +
                              bcolors.FAIL + "remote process exited with exit status " + str(exit_status) + bcolors.ENDC)
                        process['status'] = 'finished'

                # Kill some time and free up CPU before starting the next sweep
                # through the processes.
                sleep(1.0)

            # end while(1)

        except KeyboardInterrupt:
            pass   # Return execution to the calling process

    def add_worker(self, address):
        self.workers += [ merge(start_worker(self.logdir, self.center_addr, self.center_port, address,
                                             self.workers_per_node, self.cpus_per_worker),
                                {'address': address, 'worker_id': worker_id})
                          for worker_id in range(self.workers_per_node)]

    def shutdown(self):

        # Close down sockets
        for d in self.workers:
            d['stdout'].channel.close()
            d['stderr'].channel.close()

        self.center['stdout'].channel.close()
        self.center['stderr'].channel.close()

        # Close down ssh connections
        for d in self.workers:
            d['client'].close()  # Calling this multiple times on the same connection seems ok.

        self.center['client'].close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.cache import get_cache
from restlib import resources
from django.http import HttpResponse
from pyPdf import PdfFileWriter, PdfFileReader
from email.utils import parsedate_tz, mktime_tz
from email.utils import formatdate
import time
import StringIO
import requests

cache_enabled = False
if settings.CACHES.has_key("pdfappend"):
    cache_enabled = True

headers = lambda h: dict((attr,h.get(v))
    for attr, v in
    [("If-None-Match", "etag"), ("If-Modified-Since", "date")]
    if h and h.get(v))

class PDFAppender(resources.Resource):

    # This assumes a query string is used where a pdfs param
    # is repeatedly used for each PDF we need to concat.
    # Django will create a QueryDict with a pdfs attribute
    # containing all instances of the pdfs attributes used
    # in the query string
    def GET(self, request):
        # getlist here will return a list of all the query string paramaters
        # named 'pdfs'
        urls = request.GET.getlist("pdfs")
        if not urls:
            items = request.GET.items()
            items.sort(key=lambda x: x[0])
            urls = []
            for key, value in items:
               urls.append(value)

        if cache_enabled:
            cache = get_cache('pdfappend')
            urls_cache = dict((url, hit) for url, hit in zip(urls,
                [cache.get(url) for url in urls]))

            urls_needed = dict((url,True) for url, hit in urls_cache.items() if
                    hit == None or not hit.has_key('expires'))

            urls_headers = [(url, headers(urls_cache[url]))
                    for url in urls_needed.keys()]
        else:
            urls_headers = [(url, {}) for url in urls]

        s = requests.session()
        responses = [s.get(u, prefetch=True, headers=h)
                for u, h in urls_headers]

        if cache_enabled:
            self.cache_responses(responses, cache)

        response_cache = dict((r.request.url, r) for r in responses)

        master_pdf = PdfFileWriter()

        # Iterate over each pdf and add it to the master PDF
        for url in urls:
            if cache_enabled:
                if urls_needed.has_key(url):
                    if response_cache[url].status_code == 200:
                        bytes = response_cache[url].content
                    elif response_cache[url].status_code == 304:
                        bytes = urls_cache[url]['data']
                    else:
                        # Log a failed response?
                        continue
                else:
                    bytes = urls_cache[url]['data']
            else:
                # Verify request succeeded
                if not response_cache[url].status_code == 200:
                    # Log failed response?
                    continue
                bytes = response_cache[url].content

            # pyPDF needs a file like object
            input = PdfFileReader(StringIO.StringIO(bytes))

            # Add this whole PDF to the master PDF
            for page_no in range(0, input.numPages):
                master_pdf.addPage(input.getPage(page_no))

        output = HttpResponse(content_type="application/pdf")
        master_pdf.write(output)
        return output

    def cache_responses(self, responses, cache):
        cacheable = [r for r in responses if r.status_code == 200]
        for r in cacheable:
            obj = {}
            if r.headers['Expires']:
                cache.set(r.request.url,
                    {
                        'expires': True,
                        'data': response.content
                    },
                    mktime_tz(parsedate_tz(r.headers['Expires'])) -
                    time.time())
            else:
                cache.set(r.request.url,
                    {
                        'data':r.content,
                        'etag':r.headers['Etag'],
                        'date':formatdate()
                    })
<commit_msg>Comments.<commit_after>from django.conf import settings
from django.core.cache import get_cache
from restlib import resources
from django.http import HttpResponse
from pyPdf import PdfFileWriter, PdfFileReader
from email.utils import parsedate_tz, mktime_tz
from email.utils import formatdate
import time
import StringIO
import requests

cache_enabled = False
if settings.CACHES.has_key("pdfappend"):
    cache_enabled = True

# quick little snippet figure out which http headers
# to use depending on what values are in the cache
headers = lambda h: dict((attr,h.get(v))
    for attr, v in
    [("If-None-Match", "etag"), ("If-Modified-Since", "date")]
    if h and h.get(v))

class PDFAppender(resources.Resource):

    # This assumes a query string is used where a pdfs param
    # is repeatedly used for each PDF we need to concat.
    # Django will create a QueryDict with a pdfs attribute
    # containing all instances of the pdfs attributes used
    # in the query string
    def GET(self, request):
        # getlist here will return a list of all the query string paramaters
        # named 'pdfs'
        urls = request.GET.getlist("pdfs")
        if not urls:
            items = request.GET.items()
            items.sort(key=lambda x: x[0])
            urls = []
            for key, value in items:
               urls.append(value)

        if cache_enabled:
            cache = get_cache('pdfappend')
            # Create dictionary of url:<cache value>
            urls_cache = dict((url, hit) for url, hit in zip(urls,
                [cache.get(url) for url in urls]))
            # Determine which urls need to be requested
            urls_needed = dict((url,True) for url, hit in urls_cache.items() if
                    hit == None or not hit.has_key('expires'))
            # Create array of [(url, <header dict>)] for urls that need to be
            # requested
            urls_headers = [(url, headers(urls_cache[url]))
                    for url in urls_needed.keys()]
        else:
            urls_headers = [(url, {}) for url in urls]

        s = requests.session()
        responses = [s.get(u, prefetch=True, headers=h)
                for u, h in urls_headers]

        if cache_enabled:
            self.cache_responses(responses, cache)

        response_cache = dict((r.request.url, r) for r in responses)

        master_pdf = PdfFileWriter()

        # Iterate over each pdf and add it to the master PDF
        for url in urls:
            if cache_enabled:
                if urls_needed.has_key(url):
                    if response_cache[url].status_code == 200:
                        bytes = response_cache[url].content
                    elif response_cache[url].status_code == 304:
                        bytes = urls_cache[url]['data']
                    else:
                        # Log a failed response?
                        continue
                else:
                    bytes = urls_cache[url]['data']
            else:
                # Verify request succeeded
                if not response_cache[url].status_code == 200:
                    # Log failed response?
                    continue
                bytes = response_cache[url].content

            # pyPDF needs a file like object
            input = PdfFileReader(StringIO.StringIO(bytes))

            # Add this whole PDF to the master PDF
            for page_no in range(0, input.numPages):
                master_pdf.addPage(input.getPage(page_no))

        output = HttpResponse(content_type="application/pdf")
        master_pdf.write(output)
        return output

    def cache_responses(self, responses, cache):
        cacheable = [r for r in responses if r.status_code == 200]
        for r in cacheable:
            obj = {}
            if r.headers['Expires']:
                cache.set(r.request.url,
                    {
                        'expires': True,
                        'data': response.content
                    },
                    mktime_tz(parsedate_tz(r.headers['Expires'])) -
                    time.time())
            else:
                cache.set(r.request.url,
                    {
                        'data':r.content,
                        'etag':r.headers['Etag'],
                        'date':formatdate()
                    })
<|endoftext|>


<commit_before>import sys

import click

from ..cli import cli_command
from ..stacks import StackPlan
from .. import build

@click.option('--release-name')
@click.option('--from-profile')
@cli_command('release', max_apps=1)
def release_cmd(ctx, release_name=None, from_profile=None):
    """Release an application to preview, staging or production.

    If releasing to preview:
        create a new release tag and push the artefact up.

    If releasing to staging:
        copy the artefact over from the development profile.

    If releasing to production:
        just promote the current staging releasee to production.

    """
    repository_path = build.clone_or_update(ctx.stacks[ctx.apps[0]].repo_url)
    if ctx.stage == "preview":
        success = release_to_preview(ctx, repository_path)
    elif ctx.stage == "staging":
        success = release_to_staging(ctx, repository_path,
                                     release_name, from_profile)
    elif ctx.stage == "production":
        success = release_to_production(ctx, repository_path)
    else:
        raise ValueError("Invalid stage for release {}".format(ctx.stage))

    if not success:
        sys.exit(1)
    build.push_deployed_to_tag(repository_path, ctx.stage)

def get_deploy(ctx, repository_path):
    app = build.get_application_name(repository_path)
    ctx.add_apps(app)

    return StackPlan.from_ctx(ctx).get_deploy(repository_path)

def release_to_preview(ctx, repository_path):
    deploy = get_deploy(ctx, repository_path)

    release_name = build.get_release_name_for_repo(repository_path)
    if build.tag_exists(repository_path, release_name):
        raise ValueError("Already have a tag for {}".format(release_name))

    build.push_tag(repository_path, release_name)
    version, created = deploy.create_version(release_name)

    return deploy.deploy(version, ctx.stage)

def release_to_staging(ctx, repository_path, release_name, from_profile):
    if release_name is None:
        raise ValueError("Release name required for staging release")
    if from_profile is None:
        raise ValueError("Source profile required for staging release")

    deploy = get_deploy(ctx, repository_path)

    if not deploy.version_exists(release_name):
        source_plan = StackPlan.from_ctx(ctx,
                                         stage='preview',
                                         environment='master',
                                         profile_name=from_profile)
        source_deploy = source_plan.get_deploy(repository_path)
        package_path = source_deploy.download_package(release_name)

        deploy.create_version(release_name, from_file=package_path)

    return deploy.deploy(release_name, ctx.stage)

def release_to_production(ctx, repository_path):
    deploy = get_deploy(ctx, repository_path)

    release_tag = build.get_release_name_for_tag(repository_path,
                                                 'deployed-to-staging')
    if release_tag is None:
        raise StandardError("Could not find release tag for staging")

    return deploy.deploy(release_tag, ctx.stage)
<commit_msg>Push release tag after the app version has been created<commit_after>import sys

import click

from ..cli import cli_command
from ..stacks import StackPlan
from .. import build

@click.option('--release-name')
@click.option('--from-profile')
@cli_command('release', max_apps=1)
def release_cmd(ctx, release_name=None, from_profile=None):
    """Release an application to preview, staging or production.

    If releasing to preview:
        create a new release tag and push the artefact up.

    If releasing to staging:
        copy the artefact over from the development profile.

    If releasing to production:
        just promote the current staging releasee to production.

    """
    repository_path = build.clone_or_update(ctx.stacks[ctx.apps[0]].repo_url)
    if ctx.stage == "preview":
        success = release_to_preview(ctx, repository_path)
    elif ctx.stage == "staging":
        success = release_to_staging(ctx, repository_path,
                                     release_name, from_profile)
    elif ctx.stage == "production":
        success = release_to_production(ctx, repository_path)
    else:
        raise ValueError("Invalid stage for release {}".format(ctx.stage))

    if not success:
        sys.exit(1)
    build.push_deployed_to_tag(repository_path, ctx.stage)

def get_deploy(ctx, repository_path):
    app = build.get_application_name(repository_path)
    ctx.add_apps(app)

    return StackPlan.from_ctx(ctx).get_deploy(repository_path)

def release_to_preview(ctx, repository_path):
    deploy = get_deploy(ctx, repository_path)

    release_name = build.get_release_name_for_repo(repository_path)
    if build.tag_exists(repository_path, release_name):
        raise ValueError("Already have a tag for {}".format(release_name))

    version, created = deploy.create_version(release_name)
    build.push_tag(repository_path, release_name)

    return deploy.deploy(version, ctx.stage)

def release_to_staging(ctx, repository_path, release_name, from_profile):
    if release_name is None:
        raise ValueError("Release name required for staging release")
    if from_profile is None:
        raise ValueError("Source profile required for staging release")

    deploy = get_deploy(ctx, repository_path)

    if not deploy.version_exists(release_name):
        source_plan = StackPlan.from_ctx(ctx,
                                         stage='preview',
                                         environment='master',
                                         profile_name=from_profile)
        source_deploy = source_plan.get_deploy(repository_path)
        package_path = source_deploy.download_package(release_name)

        deploy.create_version(release_name, from_file=package_path)

    return deploy.deploy(release_name, ctx.stage)

def release_to_production(ctx, repository_path):
    deploy = get_deploy(ctx, repository_path)

    release_tag = build.get_release_name_for_tag(repository_path,
                                                 'deployed-to-staging')
    if release_tag is None:
        raise StandardError("Could not find release tag for staging")

    return deploy.deploy(release_tag, ctx.stage)
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.cache import get_cache
from restlib import resources
from django.http import HttpResponse
from pyPdf import PdfFileWriter, PdfFileReader
from email.utils import parsedate_tz, mktime_tz
from email.utils import formatdate
from urlparse import urlparse
import Queue
import time
import StringIO
import urllib3
import workerpool

CONNECTIONS_PER_HOST=6
MAX_THREADS = 12

cache_enabled = False

if settings.CACHES.has_key("pdfappend"):
    cache_enabled = True

# quick little snippet figure out which http headers
# to use depending on what values are in the cache
headers = lambda h: dict((attr,h.get(v))
    for attr, v in
    [("If-None-Match", "etag"), ("If-Modified-Since", "date")]
    if h and h.get(v))

class GetFile(workerpool.Job):
    def __init__(self, queue, connection_pool, url, headers):
        self.queue = queue
        self.pool = connection_pool
        self.url = url
        self.headers = headers

    def run(self):
        response = self.pool.request("GET", url, headers=self.headers)
        queue.put((url, response))

class PDFAppender(resources.Resource):
    # This assumes a query string is used where a pdfs param
    # is repeatedly used for each PDF we need to concat.
    # Django will create a QueryDict with a pdfs attribute
    # containing all instances of the pdfs attributes used
    # in the query string

    def GET(self, request):
        # getlist here will return a list of all the query string paramaters
        # named 'pdfs'
        urls = request.GET.getlist("pdfs")
        if not urls:
            items = request.GET.items()
            items.sort(key=lambda x: x[0])
            urls = []
            for key, value in items:
               urls.append(value)

        if cache_enabled:
            cache = get_cache('pdfappend')
            # Create dictionary of url:<cache value>
            urls_cache = dict((url, hit) for url, hit in zip(urls,
                [cache.get(url) for url in urls]))
            # Determine which urls need to be requested
            urls_needed = dict((url,True) for url, hit in urls_cache.items() if
                    hit == None or not hit.has_key('expires'))
            # Create array of [(url, <header dict>)] for urls that need to be
            # requested
            urls_headers = [(url, headers(urls_cache[url]))
                    for url in urls_needed.keys()]
        else:
            urls_headers = [(url, {}) for url in urls]

        hosts = self.sortByHost(urls_headers)

        # A couple of things here. If we have more than 1 host, we will spawn
        # a thread, but if there is only one host, we may refrain depending on
        # the following:
        # If connections per host is 1, we won't spawn a thread
        # If there are <= 3 URLS being requested, we will not spawn a thread,
        # just use a keepalive HTTP session and make the requests sequentially
        if (len(hosts)>1 or len(hosts[0]) > 3):
            self.getConcurrently(hosts)
        else:
            self.getSequential(hosts[0])

        if cache_enabled:
            self.cache_responses(responses, cache)

        response_cache = dict((r.request.url, r) for r in responses)

        master_pdf = PdfFileWriter()

        # Iterate over each pdf and add it to the master PDF
        for url in urls:
            if cache_enabled:
                if urls_needed.has_key(url):
                    if response_cache[url].status == 200:
                        bytes = response_cache[url].data
                    elif response_cache[url].status == 304:
                        bytes = urls_cache[url]['data']
                    else:
                        # Log a failed response?
                        continue
                else:
                    bytes = urls_cache[url]['data']
            else:
                # Verify request succeeded
                if not response_cache[url].status == 200:
                    # Log failed response?
                    continue
                bytes = response_cache[url].data

            # pyPDF needs a file like object
            input = PdfFileReader(StringIO.StringIO(bytes))

            # Add this whole PDF to the master PDF
            for page_no in range(0, input.numPages):
                master_pdf.addPage(input.getPage(page_no))

        output = HttpResponse(content_type="application/pdf")
        master_pdf.write(output)
        return output

    def cache_responses(self, responses, cache):
        cacheable = [(u, r) for u, r in responses if r.status == 200]
        for u, r in cacheable:
            obj = {}
            if r.headers['Expires']:
                cache.set(u, {
                    'expires': True,
                    'data': r. data
                    },
                    mktime_tz(parsedate_tz(r.headers['Expires'])) -
                    time.time()
                )
            else:
                cache.set(u, {
                    'data':r.data,
                    'etag':r.headers.get('Etag', None),
                    'date':formatdate()
                })

    def sortByHost(self, urls_headers):
        hosts = {}
        for url, header in urls_headers:
            parts = urlparse(url)
            hosts.setdefault(parts.netloc, []).append((url,header))
        return hosts.items()

    def getConcurrent(self, hosts):
        job_pool = workerpool.WorkerPool(size=MAX_THREADS)
        results = Queue.Queue()

        for host in hosts:
            conn_pool = urllib3.connection_from_url(host[0][0],
                    max_size=CONNECTIONS_PER_HOST)
            for url, headers in hosts:
                job = GetFile(results, conn_pool, url, headers)
                job.put(job)

        job_pool.shutdown()
        job_pool.join()

        responses = []
        while (not results.empty()):
            responses.append(results.get())
        return responses

    def getSequential(self, urls_headers):
        conn_pool = urllib3.connection_from_url(urls_headers[0][0],
                max_size=CONNECTIONS_PER_HOST)
        responses = []

        for url, headers in urls_headers:
            responses.append(conn_pool.request("GET", url, headers=headers))
        return responses

<commit_msg>max_size param -> maxsize.<commit_after>from django.conf import settings
from django.core.cache import get_cache
from restlib import resources
from django.http import HttpResponse
from pyPdf import PdfFileWriter, PdfFileReader
from email.utils import parsedate_tz, mktime_tz
from email.utils import formatdate
from urlparse import urlparse
import Queue
import time
import StringIO
import urllib3
import workerpool

CONNECTIONS_PER_HOST=6
MAX_THREADS = 12

cache_enabled = False

if settings.CACHES.has_key("pdfappend"):
    cache_enabled = True

# quick little snippet figure out which http headers
# to use depending on what values are in the cache
headers = lambda h: dict((attr,h.get(v))
    for attr, v in
    [("If-None-Match", "etag"), ("If-Modified-Since", "date")]
    if h and h.get(v))

class GetFile(workerpool.Job):
    def __init__(self, queue, connection_pool, url, headers):
        self.queue = queue
        self.pool = connection_pool
        self.url = url
        self.headers = headers

    def run(self):
        response = self.pool.request("GET", url, headers=self.headers)
        queue.put((url, response))

class PDFAppender(resources.Resource):
    # This assumes a query string is used where a pdfs param
    # is repeatedly used for each PDF we need to concat.
    # Django will create a QueryDict with a pdfs attribute
    # containing all instances of the pdfs attributes used
    # in the query string

    def GET(self, request):
        # getlist here will return a list of all the query string paramaters
        # named 'pdfs'
        urls = request.GET.getlist("pdfs")
        if not urls:
            items = request.GET.items()
            items.sort(key=lambda x: x[0])
            urls = []
            for key, value in items:
               urls.append(value)

        if cache_enabled:
            cache = get_cache('pdfappend')
            # Create dictionary of url:<cache value>
            urls_cache = dict((url, hit) for url, hit in zip(urls,
                [cache.get(url) for url in urls]))
            # Determine which urls need to be requested
            urls_needed = dict((url,True) for url, hit in urls_cache.items() if
                    hit == None or not hit.has_key('expires'))
            # Create array of [(url, <header dict>)] for urls that need to be
            # requested
            urls_headers = [(url, headers(urls_cache[url]))
                    for url in urls_needed.keys()]
        else:
            urls_headers = [(url, {}) for url in urls]

        hosts = self.sortByHost(urls_headers)

        # A couple of things here. If we have more than 1 host, we will spawn
        # a thread, but if there is only one host, we may refrain depending on
        # the following:
        # If connections per host is 1, we won't spawn a thread
        # If there are <= 3 URLS being requested, we will not spawn a thread,
        # just use a keepalive HTTP session and make the requests sequentially
        if (len(hosts)>1 or len(hosts[0]) > 3):
            self.getConcurrently(hosts)
        else:
            self.getSequential(hosts[0])

        if cache_enabled:
            self.cache_responses(responses, cache)

        response_cache = dict((r.request.url, r) for r in responses)

        master_pdf = PdfFileWriter()

        # Iterate over each pdf and add it to the master PDF
        for url in urls:
            if cache_enabled:
                if urls_needed.has_key(url):
                    if response_cache[url].status == 200:
                        bytes = response_cache[url].data
                    elif response_cache[url].status == 304:
                        bytes = urls_cache[url]['data']
                    else:
                        # Log a failed response?
                        continue
                else:
                    bytes = urls_cache[url]['data']
            else:
                # Verify request succeeded
                if not response_cache[url].status == 200:
                    # Log failed response?
                    continue
                bytes = response_cache[url].data

            # pyPDF needs a file like object
            input = PdfFileReader(StringIO.StringIO(bytes))

            # Add this whole PDF to the master PDF
            for page_no in range(0, input.numPages):
                master_pdf.addPage(input.getPage(page_no))

        output = HttpResponse(content_type="application/pdf")
        master_pdf.write(output)
        return output

    def cache_responses(self, responses, cache):
        cacheable = [(u, r) for u, r in responses if r.status == 200]
        for u, r in cacheable:
            obj = {}
            if r.headers['Expires']:
                cache.set(u, {
                    'expires': True,
                    'data': r. data
                    },
                    mktime_tz(parsedate_tz(r.headers['Expires'])) -
                    time.time()
                )
            else:
                cache.set(u, {
                    'data':r.data,
                    'etag':r.headers.get('Etag', None),
                    'date':formatdate()
                })

    def sortByHost(self, urls_headers):
        hosts = {}
        for url, header in urls_headers:
            parts = urlparse(url)
            hosts.setdefault(parts.netloc, []).append((url,header))
        return hosts.items()

    def getConcurrent(self, hosts):
        job_pool = workerpool.WorkerPool(size=MAX_THREADS)
        results = Queue.Queue()

        for host in hosts:
            conn_pool = urllib3.connection_from_url(host[0][0],
                    maxsize=CONNECTIONS_PER_HOST)
            for url, headers in hosts:
                job = GetFile(results, conn_pool, url, headers)
                job.put(job)

        job_pool.shutdown()
        job_pool.join()

        responses = []
        while (not results.empty()):
            responses.append(results.get())
        return responses

    def getSequential(self, urls_headers):
        conn_pool = urllib3.connection_from_url(urls_headers[0][0],
                maxsize=CONNECTIONS_PER_HOST)
        responses = []

        for url, headers in urls_headers:
            responses.append(conn_pool.request("GET", url, headers=headers))
        return responses

<|endoftext|>


<commit_before>"""Updates atom feeds."""
import datetime
import time

import dnstwist
import repository
import tools

# Time in seconds between re-processing a domain.
PERIOD = 86400

# Multiplier on period to unregister if not read.
UNREGISTER = 7

if __name__ == '__main__':
    while True:

        domains_iter = repository.iregistered_domains()

        while True:
            try:
                domain = domains_iter.next()
            except StopIteration:
                break

            if dnstwist.validate_domain(domain) is None:
                # TODO: Need to handle this appropriately. Maybe unregister?
                continue

            # Unregister long-time unread domains
            last_read = repository.delta_report_last_read(domain)
            if last_read is None:
                repository.mark_delta_report_as_read(domain)
            else:
                age = datetime.datetime.now() - last_read
                if age > datetime.timedelta(seconds=PERIOD*UNREGISTER):
                    print 'Unregistering {}'.format(domain)
                    repository.unregister_domain(domain)

            # Skip domains that have been recently updated
            delta_last_updated = repository.delta_report_updated(domain)
            if delta_last_updated is not None:
                age = datetime.datetime.now() - delta_last_updated
                if age < datetime.timedelta(seconds=PERIOD):
                    print 'Skipping {}'.format(domain)
                    continue

            start = time.time()

            existing_report = repository.get_resolution_report(domain)

            if existing_report is None:
                existing_report = {}

            new_report = {}
            for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
                ip, error = tools.resolve(entry['domain-name'])
                if error or not ip or ip is None:
                    continue
                new_report[entry['domain-name']] = ip

            repository.update_resolution_report(domain, new_report)

            delta_report = {'new': [], 'updated': [], 'deleted': []}
            for (dom, ip) in new_report.items():
                if dom in existing_report.keys():
                    if ip != existing_report[dom]:
                        delta_report['updated'].append(
                            (dom, existing_report[dom], ip)
                        )
                else:
                    delta_report['new'].append((dom, ip))
            for (dom, ip) in existing_report.items():
                if dom not in new_report.keys():
                    delta_report['deleted'].append(dom)

            repository.update_delta_report(domain, delta_report)

            print 'Updated deltas for {} in {} seconds'.format(
                domain, time.time() - start
            )

        time.sleep(60)
<commit_msg>Missed important loop in unregistration #20<commit_after>"""Updates atom feeds."""
import datetime
import time

import dnstwist
import repository
import tools

# Time in seconds between re-processing a domain.
PERIOD = 86400

# Multiplier on period to unregister if not read.
UNREGISTER = 7

if __name__ == '__main__':
    while True:

        domains_iter = repository.iregistered_domains()

        while True:
            try:
                domain = domains_iter.next()
            except StopIteration:
                break

            if dnstwist.validate_domain(domain) is None:
                # TODO: Need to handle this appropriately. Maybe unregister?
                continue

            # Unregister long-time unread domains
            last_read = repository.delta_report_last_read(domain)
            if last_read is None:
                repository.mark_delta_report_as_read(domain)
            else:
                age = datetime.datetime.now() - last_read
                if age > datetime.timedelta(seconds=PERIOD*UNREGISTER):
                    print 'Unregistering {}'.format(domain)
                    repository.unregister_domain(domain)
                    continue

            # Skip domains that have been recently updated
            delta_last_updated = repository.delta_report_updated(domain)
            if delta_last_updated is not None:
                age = datetime.datetime.now() - delta_last_updated
                if age < datetime.timedelta(seconds=PERIOD):
                    print 'Skipping {}'.format(domain)
                    continue

            start = time.time()

            existing_report = repository.get_resolution_report(domain)

            if existing_report is None:
                existing_report = {}

            new_report = {}
            for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
                ip, error = tools.resolve(entry['domain-name'])
                if error or not ip or ip is None:
                    continue
                new_report[entry['domain-name']] = ip

            repository.update_resolution_report(domain, new_report)

            delta_report = {'new': [], 'updated': [], 'deleted': []}
            for (dom, ip) in new_report.items():
                if dom in existing_report.keys():
                    if ip != existing_report[dom]:
                        delta_report['updated'].append(
                            (dom, existing_report[dom], ip)
                        )
                else:
                    delta_report['new'].append((dom, ip))
            for (dom, ip) in existing_report.items():
                if dom not in new_report.keys():
                    delta_report['deleted'].append(dom)

            repository.update_delta_report(domain, delta_report)

            print 'Updated deltas for {} in {} seconds'.format(
                domain, time.time() - start
            )

        time.sleep(60)
<|endoftext|>