<commit_before>import json
import logging
import random
import sys
import tornado
import urllib
from tornado.options import define, options
from tornado.web import RequestHandler, Application

define('debug', default=1, help='hot deployment. use in dev only', type=int)
define('port', default=8000, help='run on the given port', type=int)

# Custom backend account settings
sys.path.append("/home/anand/Downloads/devbox_configs/")
import backend

MIN_EXP_TIME = 30 * 24 * 60 * 60     # Expire after 30 days

REDIRECT_COUNTS_KEY = 'url:shorturl:resolved'
HLL_ORIG_URL_KEY = 'url:hyperloglog:original'

DOMAIN_REGEX = '[a-zA-Z\d-]{,63}(\.[a-zA-Z\d-]{,63})*'

#  TODO: try using zmq  based ioloop instead might be more useful
#  TODO: add that ConsistentHashRing setup to enable redis cluster
#  TODO: Read up on hashing algorithms and pick best suited one for url
#  shortening service. see
def validate_safe_url(url):
    #  TODO: Sanitize the incoming url, for malicious js.
    return  True
class UrlShortener(object):
    # From RFC 1738 allowed url chars
    LOWALPHA       = [ 'a', 'b','c', 'd' , 'e' , 'f' , 'g' , 'h' ,
                     'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , 'p' ,
                     'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , 'x' ,
                     'y' , 'z' ]
    HIALPHA        = [ 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , 'H' ,
                     'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , 'P' ,
                     'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , 'X' ,
                     'Y' , 'Z' ]
    DIGIT          = [ '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' ,
                     '8' , '9' ]
    SAFE           = [ '$' , '-' , '_' , '.' , '+' ]
    EXTRA          = [ '!' , '*' , "'" , '(' , ')' , ',' ]
    PUNCTUATION    = [ '<' , '>' , '#' , '%' , "'" ]

    URL_ALLOWED_CHARS = LOWALPHA +\
                        HIALPHA +\
                        DIGIT +\
                        SAFE +\
                        EXTRA +\
                        PUNCTUATION

    def __init__(self):
        self.redis = backend.redisConn

    def get_stats(self):
        urls_count = self.redis.pfcount(HLL_ORIG_URL_KEY) if self.redis.pfcount(HLL_ORIG_URL_KEY) else 0
        redirects = self.redis.get(REDIRECT_COUNTS_KEY) if self.redis.get(REDIRECT_COUNTS_KEY) else 0
        stats = {r'urls':urls_count,
                 r'redirects':redirects
                 }
        return stats

    def shorten_url(self, url):
        safe_url = validate_safe_url(url)
        # Check if the given url is a shortened url. stop malicious programs from inducing a redirect loop
        if not safe_url:
            logging.warn("#urlshortener: short_url provided as input for shortening")
            return None
        orig_url_not_exists = self.redis.pfadd(HLL_ORIG_URL_KEY, url)
        if orig_url_not_exists:
            short_url = "".join([random.choice(self.URL_ALLOWED_CHARS) for i in range(5)])
            if not self.redis.get(short_url):
                self.redis.setex(short_url, url, MIN_EXP_TIME)
                self.redis.setex(url, short_url, MIN_EXP_TIME)
            else:
                # Since collisions are possible, this means there was a
                # collision
                logging.warn("#urlshortener: Collision Orig Url: %s, generated short url: %s" %(url, short_url))
                self.shorten_url(url)
        else:
            # Original url already shortenede, just return th
            short_url = self.redis.get(url)
        return short_url

    def retrieve_orig_url(self, short_url):
        return str(self.redis.get(short_url))

class ShortUrlHandler(RequestHandler):
    def get(self):
        self.post()

    def post(self):
        assert self.request.uri.startswith('/url')
        url_parts = self.request.uri.split('/')
        short_url = url_parts[2] if (url_parts) > 2 else None
        if short_url:
            logging.info('# Received short url: %s' % short_url)
            orig_url = url_shortener.retrieve_orig_url(short_url)
            url_shortener.redis.incrby(REDIRECT_COUNTS_KEY, 1)
            self.redirect(orig_url)
        else:
            self.redirect('/url/shorten')

class ShortenUrlHandler(RequestHandler):
    def get(self):
        self.render('static/index.html')
    def post(self):
        orig_url = self.get_argument('orig_url')
        logging.info('# Received Original url: %s' % orig_url)
        short_url = url_shortener.shorten_url(orig_url)
        if short_url:
            linkified_short_url = 'Click Here'
            self.finish(json.dumps({'url': linkified_short_url}, ensure_ascii=False).encode('utf-8'))

class StatsHandler(RequestHandler):
    def get(self):
        self.finish(json.dumps(url_shortener.get_stats(), ensure_ascii=False).encode('utf-8'))

class Application(Application):
    #  """
    #  >>> import requests
    #  >>> requests.post("/shorten", params={"orig_url":"http://google.com"})
    #  >>> resp = requests.get("/shorten", params={"short_url": "265477614567132497141480353139365708304L"})
    #  >>> assert resp.url=="http://google.com"
    #  """
    def __init__(self):
        handlers = [
                (r'/url/shorten', ShortenUrlHandler),
                (r'/url/stats',StatsHandler),
                (r'/url/(?!stats|shorten).*', ShortUrlHandler),
                ]
        settings = dict(
            autoescape=None,  # tornado 2.1 backward compatibility
            debug=options.debug,
            gzip=True,
            )
        settings.update({'static_path':'./static'})
        tornado.web.Application.__init__(self, handlers, **settings)

def main():
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port, xheaders=True)
    loop = tornado.ioloop.IOLoop.instance()
    loop.start()

if __name__ == '__main__':
    url_shortener = UrlShortener()
    main()
<commit_msg>unquote shorturls before redirecting.<commit_after>import json
import logging
import random
import sys
import tornado
import urllib
from tornado.options import define, options
from tornado.web import RequestHandler, Application

define('debug', default=1, help='hot deployment. use in dev only', type=int)
define('port', default=8000, help='run on the given port', type=int)

# Custom backend account settings
sys.path.append("/home/anand/Downloads/devbox_configs/")
import backend

MIN_EXP_TIME = 30 * 24 * 60 * 60     # Expire after 30 days

REDIRECT_COUNTS_KEY = 'url:shorturl:resolved'
HLL_ORIG_URL_KEY = 'url:hyperloglog:original'

DOMAIN_REGEX = '[a-zA-Z\d-]{,63}(\.[a-zA-Z\d-]{,63})*'

#  TODO: try using zmq  based ioloop instead might be more useful
#  TODO: add that ConsistentHashRing setup to enable redis cluster
#  TODO: Read up on hashing algorithms and pick best suited one for url
#  shortening service. see
def validate_safe_url(url):
    #  TODO: Sanitize the incoming url, for malicious js.
    return  True
class UrlShortener(object):
    # From RFC 1738 allowed url chars
    LOWALPHA       = [ 'a', 'b','c', 'd' , 'e' , 'f' , 'g' , 'h' ,
                     'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , 'p' ,
                     'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , 'x' ,
                     'y' , 'z' ]
    HIALPHA        = [ 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , 'H' ,
                     'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , 'P' ,
                     'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , 'X' ,
                     'Y' , 'Z' ]
    DIGIT          = [ '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' ,
                     '8' , '9' ]
    SAFE           = [ '$' , '-' , '_' , '.' , '+' ]
    EXTRA          = [ '!' , '*' , "'" , '(' , ')' , ',' ]
    PUNCTUATION    = [ '<' , '>' , '#' , '%' , "'" ]

    URL_ALLOWED_CHARS = LOWALPHA +\
                        HIALPHA +\
                        DIGIT +\
                        SAFE +\
                        EXTRA +\
                        PUNCTUATION

    def __init__(self):
        self.redis = backend.redisConn

    def get_stats(self):
        urls_count = self.redis.pfcount(HLL_ORIG_URL_KEY) if self.redis.pfcount(HLL_ORIG_URL_KEY) else 0
        redirects = self.redis.get(REDIRECT_COUNTS_KEY) if self.redis.get(REDIRECT_COUNTS_KEY) else 0
        stats = {r'urls':urls_count,
                 r'redirects':redirects
                 }
        return stats

    def shorten_url(self, url):
        safe_url = validate_safe_url(url)
        # Check if the given url is a shortened url. stop malicious programs from inducing a redirect loop
        if not safe_url:
            logging.warn("#urlshortener: short_url provided as input for shortening")
            return None
        orig_url_not_exists = self.redis.pfadd(HLL_ORIG_URL_KEY, url)
        if orig_url_not_exists:
            short_url = "".join([random.choice(self.URL_ALLOWED_CHARS) for i in range(5)])
            if not self.redis.get(short_url):
                self.redis.setex(short_url, url, MIN_EXP_TIME)
                self.redis.setex(url, short_url, MIN_EXP_TIME)
            else:
                # Since collisions are possible, this means there was a
                # collision
                logging.warn("#urlshortener: Collision Orig Url: %s, generated short url: %s" %(url, short_url))
                self.shorten_url(url)
        else:
            # Original url already shortenede, just return th
            short_url = self.redis.get(url)
        return short_url

    def retrieve_orig_url(self, short_url):
        return str(self.redis.get(short_url))

class ShortUrlHandler(RequestHandler):
    def get(self):
        self.post()

    def post(self):
        assert self.request.uri.startswith('/url')
        url_parts = self.request.uri.split('/')
        short_url = urllib.unquote_plus(url_parts[2]) if (url_parts) > 2 else None
        if short_url:
            logging.info('# Received short url: %s' % short_url)
            orig_url = url_shortener.retrieve_orig_url(short_url)
            url_shortener.redis.incrby(REDIRECT_COUNTS_KEY, 1)
            self.redirect(orig_url)
        else:
            self.redirect('/url/shorten')

class ShortenUrlHandler(RequestHandler):
    def get(self):
        self.render('static/index.html')
    def post(self):
        orig_url = self.get_argument('orig_url')
        logging.info('# Received Original url: %s' % orig_url)
        short_url = url_shortener.shorten_url(orig_url)
        if short_url:
            linkified_short_url = 'Click Here'
            self.finish(json.dumps({'url': linkified_short_url}, ensure_ascii=False).encode('utf-8'))

class StatsHandler(RequestHandler):
    def get(self):
        self.finish(json.dumps(url_shortener.get_stats(), ensure_ascii=False).encode('utf-8'))

class Application(Application):
    #  """
    #  >>> import requests
    #  >>> requests.post("/shorten", params={"orig_url":"http://google.com"})
    #  >>> resp = requests.get("/shorten", params={"short_url": "265477614567132497141480353139365708304L"})
    #  >>> assert resp.url=="http://google.com"
    #  """
    def __init__(self):
        handlers = [
                (r'/url/shorten', ShortenUrlHandler),
                (r'/url/stats',StatsHandler),
                (r'/url/(?!stats|shorten).*', ShortUrlHandler),
                ]
        settings = dict(
            autoescape=None,  # tornado 2.1 backward compatibility
            debug=options.debug,
            gzip=True,
            )
        settings.update({'static_path':'./static'})
        tornado.web.Application.__init__(self, handlers, **settings)

def main():
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port, xheaders=True)
    loop = tornado.ioloop.IOLoop.instance()
    loop.start()

if __name__ == '__main__':
    url_shortener = UrlShortener()
    main()
<|endoftext|>


<commit_before># Author: Michael Lissner
# Date created: 2013-06-21

from datetime import date
from datetime import datetime

from juriscraper.OpinionSite import OpinionSite
from lxml import html

class Site(OpinionSite):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.court_id = self.__module__
        today = date.today()
        today = date(2016, 4, 5)
        self.crawl_date = today
        self.url = 'http://supremecourt.ne.gov/coa/opinions/%s' % today.strftime('%Y-%m-%d')
        #self.url = 'http://supremecourt.ne.gov/coa/opinions/2013-06-18'  # For testing...
        self.back_scrape_iterable = range(0, 11)

    def _get_download_urls(self):
        path = '//tr[contains(@class, "opinion")]//a/@href'
        return list(self.html.xpath(path))

    def _get_case_names(self):
        path = '//tr[contains(@class, "opinion")]//a/text()'
        return list(self.html.xpath(path))

    def _get_case_dates(self):
        # Uses the case names path as surrogate for date counting
        path = '//tr[contains(@class, "opinion")]//a/text()'
        return [self.crawl_date] * len(self.html.xpath(path))

    def _get_precedential_statuses(self):
        path = '//tr[contains(@class, "opinion")]//a/@href'
        statuses = []
        for url in self.html.xpath(path):
            if 'memorandum' in url:
                statuses.append("Unpublished")
            else:
                statuses.append("Published")
        return statuses

    def _get_docket_numbers(self):
        path = '//tr[contains(@class, "opinion")]/td[1]//text()[normalize-space() != ""]'
        return list(self.html.xpath(path))

    def _download_backwards(self, i):
        host = 'http://supremecourt.ne.gov'
        paths = ['/coa/opinions/2013-06-18',
                 '/coa/opinions/2013-06-04',
                 '/coa/opinions/2013-05-28',
                 '/coa/opinions/2013-05-21',
                 '/coa/opinions/2013-05-14',
                 '/coa/opinions/2013-05-07',
                 '/coa/opinions/2013-04-30',
                 '/coa/opinions/2013-04-23',
                 '/coa/opinions/2013-04-16',
                 '/coa/opinions/2013-04-09',
                 '/coa/opinions/2013-04-02',
                 '/coa/opinions/2013-03-26']
        self.crawl_date = datetime.strptime(paths[i][-10:], '%Y-%m-%d').date()
        self.url = host + paths[i]
        self.html = self._download()
<commit_msg>Removes testing code from nebctapp.<commit_after># Author: Michael Lissner
# Date created: 2013-06-21

from datetime import date
from datetime import datetime

from juriscraper.OpinionSite import OpinionSite
from lxml import html

class Site(OpinionSite):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.court_id = self.__module__
        today = date.today()
        self.crawl_date = today
        self.url = 'http://supremecourt.ne.gov/coa/opinions/%s' % today.strftime('%Y-%m-%d')
        #self.url = 'http://supremecourt.ne.gov/coa/opinions/2013-06-18'  # For testing...
        self.back_scrape_iterable = range(0, 11)

    def _get_download_urls(self):
        path = '//tr[contains(@class, "opinion")]//a/@href'
        return list(self.html.xpath(path))

    def _get_case_names(self):
        path = '//tr[contains(@class, "opinion")]//a/text()'
        return list(self.html.xpath(path))

    def _get_case_dates(self):
        # Uses the case names path as surrogate for date counting
        path = '//tr[contains(@class, "opinion")]//a/text()'
        return [self.crawl_date] * len(self.html.xpath(path))

    def _get_precedential_statuses(self):
        path = '//tr[contains(@class, "opinion")]//a/@href'
        statuses = []
        for url in self.html.xpath(path):
            if 'memorandum' in url:
                statuses.append("Unpublished")
            else:
                statuses.append("Published")
        return statuses

    def _get_docket_numbers(self):
        path = '//tr[contains(@class, "opinion")]/td[1]//text()[normalize-space() != ""]'
        return list(self.html.xpath(path))

    def _download_backwards(self, i):
        host = 'http://supremecourt.ne.gov'
        paths = ['/coa/opinions/2013-06-18',
                 '/coa/opinions/2013-06-04',
                 '/coa/opinions/2013-05-28',
                 '/coa/opinions/2013-05-21',
                 '/coa/opinions/2013-05-14',
                 '/coa/opinions/2013-05-07',
                 '/coa/opinions/2013-04-30',
                 '/coa/opinions/2013-04-23',
                 '/coa/opinions/2013-04-16',
                 '/coa/opinions/2013-04-09',
                 '/coa/opinions/2013-04-02',
                 '/coa/opinions/2013-03-26']
        self.crawl_date = datetime.strptime(paths[i][-10:], '%Y-%m-%d').date()
        self.url = host + paths[i]
        self.html = self._download()
<|endoftext|>


<commit_before>import iati
from rest_framework import serializers
from api.generics.serializers import DynamicFieldsModelSerializer

class OrganisationSerializer(DynamicFieldsModelSerializer):
    url = serializers.HyperlinkedIdentityField(view_name='organisation-detail')

    class Meta:
        model = iati.models.Organisation
        fields = (
            'url',
            'code',
            'abbreviation',
            'type',
            'reported_by_organisation',
            'name',
            'original_ref',

            # Reverse linked data
            'activity_reporting_organisation',
            'activity_set',
            'activityparticipatingorganisation_set',
            'transaction_providing_organisation',
            'transaction_receiving_organisation',
        )
<commit_msg>fixed organisation-detail url by encoding organisation.code<commit_after>import django.utils.http
import iati
from rest_framework import serializers
from api.generics.serializers import DynamicFieldsModelSerializer

class OrganisationSerializer(DynamicFieldsModelSerializer):
    class EncodedHyperlinkedIdentityField(
            serializers.HyperlinkedIdentityField):
        def get_url(self, obj, view_name, request, format):
            if obj.pk is None:
                return None
            lookup_value = getattr(obj, self.lookup_field)
            quoted_lookup_value = django.utils.http.urlquote(lookup_value)

            kwargs = {self.lookup_url_kwarg: quoted_lookup_value}
            return self.reverse(
                view_name, kwargs=kwargs, request=request, format=format)

    url = EncodedHyperlinkedIdentityField(view_name='organisation-detail')

    class Meta:
        model = iati.models.Organisation
        fields = (
            'url',
            'code',
            'abbreviation',
            'type',
            'reported_by_organisation',
            'name',
            'original_ref',

            # Reverse linked data
            'activity_reporting_organisation',
            'activity_set',
            'activityparticipatingorganisation_set',
            'transaction_providing_organisation',
            'transaction_receiving_organisation',
        )
<|endoftext|>


<commit_before>'''
Tests for the SVN state
'''
import os
import shutil
import socket
try:
    import integration
except ImportError:
    if __name__ == '__main__':
        import os
        import sys
        sys.path.insert(
            0, os.path.abspath(
                os.path.join(
                    os.path.dirname(__file__), '../../'
                )
            )
        )
    import integration

class SvnTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
    '''
    Validate the svn state
    '''

    def setUp(self):
        super(SvnTest, self).setUp()

        if not self.run_function('cmd.has_exec', ['svn']):
            self.skipTest("The executable 'svn' is not available.")

        self.__domain = 'svn.apache.org'
        try:
            if hasattr(socket, 'setdefaulttimeout'):
                # 10 second dns timeout
                socket.setdefaulttimeout(10)
            socket.gethostbyname(self.__domain)
        except socket.error:
            msg = 'error resolving {0}, possible network issue?'
            self.skipTest(msg.format(self.__domain))

        self.target = os.path.join(integration.TMP, 'apache_http_test_repo')
        self.name = 'http://{0}/repos/asf/httpd/httpd/trunk/test/'.format(self.__domain)
        self.new_rev = '1456987'

    def tearDown(self):
        shutil.rmtree(self.target, ignore_errors=True)

    def test_latest(self):
        '''
        svn.latest
        '''
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))
        self.assertSaltStateChangesEqual(
            ret, self.name, keys=['new']
        )
        self.assertSaltStateChangesEqual(
            ret, self.new_rev, keys=['revision']
        )

    def test_latest_failure(self):
        '''
        svn.latest
        '''
        ret = self.run_state(
            'svn.latest',
            name='https://youSpelledApacheWrong.com/repos/asf/httpd/httpd/trunk/test/',
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltFalseReturn(ret)
        self.assertFalse(os.path.isdir(os.path.join(self.target, '.svn')))

    def test_latest_empty_dir(self):
        '''
        svn.latest
        '''
        if not os.path.isdir(self.target):
            os.mkdir(self.target)
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

    def no_test_latest_existing_repo(self):
        '''
        svn.latest against existing repository
        '''
        current_rev = '1442865'
        cwd, basename = os.path.split(self.target)
        opts = ('-r', current_rev)

        out = self.run_function('svn.checkout',
                                [cwd, self.name, basename, None, None, opts])
        assert out

        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertSaltStateChangesEqual(
            ret,
            '{0} => {1}'.format(current_rev, self.new_rev),
            keys=['revision']
        )
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

    def no_test_latest_existing_repo_no_rev_change(self):
        '''
        svn.latest against existing repository
        '''
        current_rev = self.new_rev
        cwd, basename = os.path.split(self.target)
        opts = ('-r', current_rev)
        out = self.run_function('svn.checkout',
                                [cwd, self.name, basename, None, None, opts])
        assert out
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertSaltStateChangesEqual(ret, {})
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(SvnTest)
<commit_msg>Updated `integration.states.svn` test suite to use `salttesting.helpers.ensure_in_syspath()`<commit_after>'''
Tests for the SVN state
'''

# Import python libs
import os
import shutil
import socket

# Import Salt Testing libs
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')

# Import salt libs
import integration

class SvnTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
    '''
    Validate the svn state
    '''

    def setUp(self):
        super(SvnTest, self).setUp()

        if not self.run_function('cmd.has_exec', ['svn']):
            self.skipTest("The executable 'svn' is not available.")

        self.__domain = 'svn.apache.org'
        try:
            if hasattr(socket, 'setdefaulttimeout'):
                # 10 second dns timeout
                socket.setdefaulttimeout(10)
            socket.gethostbyname(self.__domain)
        except socket.error:
            msg = 'error resolving {0}, possible network issue?'
            self.skipTest(msg.format(self.__domain))

        self.target = os.path.join(integration.TMP, 'apache_http_test_repo')
        self.name = 'http://{0}/repos/asf/httpd/httpd/trunk/test/'.format(
            self.__domain
        )
        self.new_rev = '1456987'

    def tearDown(self):
        shutil.rmtree(self.target, ignore_errors=True)

    def test_latest(self):
        '''
        svn.latest
        '''
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))
        self.assertSaltStateChangesEqual(
            ret, self.name, keys=['new']
        )
        self.assertSaltStateChangesEqual(
            ret, self.new_rev, keys=['revision']
        )

    def test_latest_failure(self):
        '''
        svn.latest
        '''
        ret = self.run_state(
            'svn.latest',
            name='https://youSpelledApacheWrong.com/repo/asf/httpd/trunk/',
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltFalseReturn(ret)
        self.assertFalse(os.path.isdir(os.path.join(self.target, '.svn')))

    def test_latest_empty_dir(self):
        '''
        svn.latest
        '''
        if not os.path.isdir(self.target):
            os.mkdir(self.target)
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

    def no_test_latest_existing_repo(self):
        '''
        svn.latest against existing repository
        '''
        current_rev = '1442865'
        cwd, basename = os.path.split(self.target)
        opts = ('-r', current_rev)

        out = self.run_function('svn.checkout',
                                [cwd, self.name, basename, None, None, opts])
        assert out

        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertSaltStateChangesEqual(
            ret,
            '{0} => {1}'.format(current_rev, self.new_rev),
            keys=['revision']
        )
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

    def no_test_latest_existing_repo_no_rev_change(self):
        '''
        svn.latest against existing repository
        '''
        current_rev = self.new_rev
        cwd, basename = os.path.split(self.target)
        opts = ('-r', current_rev)
        out = self.run_function('svn.checkout',
                                [cwd, self.name, basename, None, None, opts])
        assert out
        ret = self.run_state(
            'svn.latest',
            name=self.name,
            rev=self.new_rev,
            target=self.target,
        )
        self.assertSaltTrueReturn(ret)
        self.assertSaltStateChangesEqual(ret, {})
        self.assertTrue(os.path.isdir(os.path.join(self.target, '.svn')))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(SvnTest)
<|endoftext|>


<commit_before>import json
import logging
import redis
import random
import tornado
import urllib
from tornado.options import define, options
from tornado.web import RequestHandler, Application

define('debug', default=1, help='hot deployment. use in dev only', type=int)
define('port', default=8000, help='run on the given port', type=int)

REDIS_IP = '127.0.0.1'
REDIS_PORT = 6379
REPLICAS_SIZE = 10  # Number of Replicas
MIN_EXP_TIME = 24 * 60 * 60     # Expire after 1 day
REDIRECT_COUNTS_KEY = 'url:shorturl:resolved'
HLL_ORIG_URL_KEY = 'url:hyperloglog:original'
SET_SHORT_URL_KEY = 'url:short:set'

def jsonify(obj, **kwargs):
    assert isinstance(obj, dict), "Only dictionaries please"
    new_obj = dict()
    for k,v in obj.items():
        if isinstance(k,str):
            nk = k.encode('ascii')
        else:
            nk = k
        print(type(nk))
        if isinstance(v,str):
            nv = v.encode('ascii')
        else:
            nv = v
        new_obj[nk] = nv
    return new_obj
    #return json.dump(new_obj, **kwargs)

#  TODO: try using zmq  based ioloop instead might be more useful
#  TODO: add that ConsistentHashRing setup to enable redis cluster
#  TODO: Read up on hashing algorithms and pick best suited one for url
#  shortening service. see
#  TODO: Sanitize the incoming url, for malicious js.

class UrlShortener(object):
    # From RFC 1738 allowed url chars
    LOWALPHA       = [ 'a', 'b','c', 'd' , 'e' , 'f' , 'g' , 'h' ,
                     'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , 'p' ,
                     'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , 'x' ,
                     'y' , 'z' ]
    HIALPHA        = [ 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , 'H' ,
                     'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , 'P' ,
                     'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , 'X' ,
                     'Y' , 'Z' ]
    DIGIT          = [ '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' ,
                     '8' , '9' ]
    SAFE           = [ '$' , '-' , '_' , '.' , '+' ]
    EXTRA          = [ '!' , '*' , "'" , '(' , ')' , ',' ]
    PUNCTUATION    = [ '<' , '>' , '#' , '%' , "'" ]

    URL_ALLOWED_CHARS = LOWALPHA +\
                        HIALPHA +\
                        DIGIT +\
                        SAFE +\
                        EXTRA +\
                        PUNCTUATION

    def __init__(self):
        self.redis = redis.Redis(host=REDIS_IP, port=REDIS_PORT)

    def get_stats(self):
        urls_count = self.redis.pfcount(HLL_ORIG_URL_KEY) if self.redis.pfcount(HLL_ORIG_URL_KEY) else 0
        redirects = self.redis.get(REDIRECT_COUNTS_KEY) if self.redis.get(REDIRECT_COUNTS_KEY) else 0
        stats = {r'urls':urls_count,
                 r'redirects':redirects
                 }
        return stats

    def shorten_url(self, url):
        orig_url_not_exists = self.redis.pfadd(HLL_ORIG_URL_KEY, url)
        # Check if the given url is a shortened url. stop malicious programs from inducing a redirect loop
        short_url_not_exists = self.redis.sadd(SET_SHORT_URL_KEY, url)

        if not short_url_not_exists:
            logging.warn("#urlshortener: short_url provided as input for shortening")
            return None
        # If execution reaches here input is not a shortened url, remove it from storage.
        self.redis.srem(SET_SHORT_URL_KEY, url)
        if orig_url_not_exists:
            short_url = "".join([random.choice(self.URL_ALLOWED_CHARS) for i in range(5)])
            if not self.redis.get(short_url):
                self.redis.setex(short_url, url, MIN_EXP_TIME)
                self.redis.setex(url, short_url, MIN_EXP_TIME)
                self.redis.sadd(SET_SHORT_URL_KEY, short_url)
            else:
                # Since collisions are possible, this means there was a
                # collision
                logging.warn("#urlshortener: Collision Orig Url: %s, generated short url: %s" %(url, short_url))
                self.shorten_url(url)
        else:
            # Original url already shortenede, just return th
            short_url = self.redis.get(url)
        return short_url

    def retrieve_orig_url(self, short_url):
        return str(self.redis.get(short_url))

class ShortUrlHandler(RequestHandler):
    def get(self, args=None):
        logging.info(args)
        if args:
            data = {"short_url": args}
            assert url_shortener.redis.get(args)
            self.request.query_arguments.update(data)
            self.post()
        else:
            self.render('static/index.html')

    def post(self):
        short_url = self.request.query_arguments.get('short_url')
        logging.info('# Received short url: %s' % short_url)
        orig_url = url_shortener.retrieve_orig_url(short_url)
        url_shortener.redis.incrby(REDIRECT_COUNTS_KEY, 1)
        self.redirect(orig_url)

class ShortenUrlHandler(RequestHandler):
    def get(self):
        self.render('static/index.html')
    def post(self):
        orig_url = self.get_argument('orig_url')
        logging.info('# Received Original url: %s' % orig_url)
        short_url = url_shortener.shorten_url(orig_url)
        if short_url:
            linkified_short_url = 'Click Here'
            self.finish(json.dumps({'url': linkified_short_url}, ensure_ascii=False).encode('utf-8'))
        else:
            self.redirect("/url/")

class StatsHandler(RequestHandler):
    def get(self):
        self.finish(json.dumps(url_shortener.get_stats(), ensure_ascii=False).encode('utf-8'))

class Application(Application):
    #  """
    #  >>> import requests
    #  >>> requests.post("/shorten", params={"orig_url":"http://google.com"})
    #  >>> resp = requests.get("/shorten", params={"short_url": "265477614567132497141480353139365708304L"})
    #  >>> assert resp.url=="http://google.com"
    #  """
    def __init__(self):
        handlers = [
                (r'/url/shorten', ShortenUrlHandler),
                (r'/url/stats',StatsHandler),
                (r'/url/(?!stats).*', ShortUrlHandler),
                ]
        settings = dict(
            autoescape=None,  # tornado 2.1 backward compatibility
            debug=options.debug,
            gzip=True,
            )
        settings.update({'static_path':'./static'})
        tornado.web.Application.__init__(self, handlers, **settings)

def main():
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port, xheaders=True)
    loop = tornado.ioloop.IOLoop.instance()
    loop.start()

if __name__ == '__main__':
    url_shortener = UrlShortener()
    main()
<commit_msg>ou more stuff<commit_after>import json
import logging
import redis
import random
import tornado
import urllib
from tornado.options import define, options
from tornado.web import RequestHandler, Application

define('debug', default=1, help='hot deployment. use in dev only', type=int)
define('port', default=8000, help='run on the given port', type=int)

REDIS_IP = '127.0.0.1'
REDIS_PORT = 6379
REPLICAS_SIZE = 10  # Number of Replicas
MIN_EXP_TIME = 24 * 60 * 60     # Expire after 1 day
REDIRECT_COUNTS_KEY = 'url:shorturl:resolved'
HLL_ORIG_URL_KEY = 'url:hyperloglog:original'

#  TODO: try using zmq  based ioloop instead might be more useful
#  TODO: add that ConsistentHashRing setup to enable redis cluster
#  TODO: Read up on hashing algorithms and pick best suited one for url
#  shortening service. see
#  TODO: Sanitize the incoming url, for malicious js.

class UrlShortener(object):
    # From RFC 1738 allowed url chars
    LOWALPHA       = [ 'a', 'b','c', 'd' , 'e' , 'f' , 'g' , 'h' ,
                     'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , 'p' ,
                     'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , 'x' ,
                     'y' , 'z' ]
    HIALPHA        = [ 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , 'H' ,
                     'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , 'P' ,
                     'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , 'X' ,
                     'Y' , 'Z' ]
    DIGIT          = [ '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' ,
                     '8' , '9' ]
    SAFE           = [ '$' , '-' , '_' , '.' , '+' ]
    EXTRA          = [ '!' , '*' , "'" , '(' , ')' , ',' ]
    PUNCTUATION    = [ '<' , '>' , '#' , '%' , "'" ]

    URL_ALLOWED_CHARS = LOWALPHA +\
                        HIALPHA +\
                        DIGIT +\
                        SAFE +\
                        EXTRA +\
                        PUNCTUATION

    def __init__(self):
        self.redis = redis.Redis(host=REDIS_IP, port=REDIS_PORT)

    def get_stats(self):
        urls_count = self.redis.pfcount(HLL_ORIG_URL_KEY) if self.redis.pfcount(HLL_ORIG_URL_KEY) else 0
        redirects = self.redis.get(REDIRECT_COUNTS_KEY) if self.redis.get(REDIRECT_COUNTS_KEY) else 0
        stats = {r'urls':urls_count,
                 r'redirects':redirects
                 }
        return stats

    def shorten_url(self, url):
        orig_url_not_exists = self.redis.pfadd(HLL_ORIG_URL_KEY, url)
        # Check if the given url is a shortened url. stop malicious programs from inducing a redirect loop
        short_url_not_exists = self.redis.get(short_url)

        if not short_url_not_exists:
            logging.warn("#urlshortener: short_url provided as input for shortening")
            return None
        if orig_url_not_exists:
            short_url = "".join([random.choice(self.URL_ALLOWED_CHARS) for i in range(5)])
            if not self.redis.get(short_url):
                self.redis.setex(short_url, url, MIN_EXP_TIME)
                self.redis.setex(url, short_url, MIN_EXP_TIME)
            else:
                # Since collisions are possible, this means there was a
                # collision
                logging.warn("#urlshortener: Collision Orig Url: %s, generated short url: %s" %(url, short_url))
                self.shorten_url(url)
        else:
            # Original url already shortenede, just return th
            short_url = self.redis.get(url)
        return short_url

    def retrieve_orig_url(self, short_url):
        return str(self.redis.get(short_url))

class ShortUrlHandler(RequestHandler):
    def get(self, args=None):
        logging.info(args)
        if args:
            data = {"short_url": args}
            assert url_shortener.redis.get(args)
            self.request.query_arguments.update(data)
            self.post()
        else:
            self.render('static/index.html')

    def post(self):
        short_url = self.request.query_arguments.get('short_url')
        logging.info('# Received short url: %s' % short_url)
        orig_url = url_shortener.retrieve_orig_url(short_url)
        url_shortener.redis.incrby(REDIRECT_COUNTS_KEY, 1)
        self.redirect(orig_url)

class ShortenUrlHandler(RequestHandler):
    def get(self):
        self.render('static/index.html')
    def post(self):
        orig_url = self.get_argument('orig_url')
        logging.info('# Received Original url: %s' % orig_url)
        short_url = url_shortener.shorten_url(orig_url)
        if short_url:
            linkified_short_url = 'Click Here'
            self.finish(json.dumps({'url': linkified_short_url}, ensure_ascii=False).encode('utf-8'))
        else:
            self.redirect("/url/")

class StatsHandler(RequestHandler):
    def get(self):
        self.finish(json.dumps(url_shortener.get_stats(), ensure_ascii=False).encode('utf-8'))

class Application(Application):
    #  """
    #  >>> import requests
    #  >>> requests.post("/shorten", params={"orig_url":"http://google.com"})
    #  >>> resp = requests.get("/shorten", params={"short_url": "265477614567132497141480353139365708304L"})
    #  >>> assert resp.url=="http://google.com"
    #  """
    def __init__(self):
        handlers = [
                (r'/url/shorten', ShortenUrlHandler),
                (r'/url/stats',StatsHandler),
                (r'/url/(?!stats).*', ShortUrlHandler),
                ]
        settings = dict(
            autoescape=None,  # tornado 2.1 backward compatibility
            debug=options.debug,
            gzip=True,
            )
        settings.update({'static_path':'./static'})
        tornado.web.Application.__init__(self, handlers, **settings)

def main():
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port, xheaders=True)
    loop = tornado.ioloop.IOLoop.instance()
    loop.start()

if __name__ == '__main__':
    url_shortener = UrlShortener()
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 Nitor Creations Oy
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
import sys
import aws_infra_util
import os
import tempfile
import collections
import time
import datetime

def deploy(stack_name, yaml_template):

    # Disable buffering, from http://stackoverflow.com/questions/107705/disable-output-buffering
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    # Assume amibakery role

    p = subprocess.Popen(['aws', 'sts', 'assume-role', '--role-arn', 'arn:aws:iam::832585949989:role/amibakery', '--role-session-name', 'amibakery-deploy'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
    output = p.communicate()
    if p.returncode:
       sys.exit("Assume role failed: " + output[1])

    # AWS Credentials

    credentials = aws_infra_util.json_load(output[0])
    aws_access_key_id = credentials['Credentials']['AccessKeyId']
    aws_secret_access_key = credentials['Credentials']['SecretAccessKey']
    aws_session_token = credentials['Credentials']['SessionToken']

    # Get AMI metadata

    describe_ami_command = [ "aws", "ec2", "describe-images", "--image-ids", os.environ["paramAmi"] ]
    print("Checking AMI " + os.environ["paramAmi"] + " metadata: " + str(describe_ami_command))
    p = subprocess.Popen(describe_ami_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    if p.returncode:
        sys.exit("Failed to retrieve ami metadata for " + os.environm["paramAmi"])

    ami_meta = aws_infra_util.json_load(output[0])
    print("Result: " + aws_infra_util.json_save(ami_meta))
    os.environ["paramAmiName"] = ami_meta['Images'][0]['Name']
    os.environ["paramAmiCreated"] = ami_meta['Images'][0]['CreationDate']

    print("\n\n**** Deploying stack '" + stack_name + "' with template '" + yaml_template + "' and ami_id " + os.environ["paramAmi"])

    # Load yaml template and import scripts and patch userdata with metadata hash & params

    template_doc = aws_infra_util.yaml_load(open(yaml_template))
    template_doc = aws_infra_util.import_scripts(template_doc, yaml_template)
    aws_infra_util.patch_launchconf_userdata_with_metadata_hash_and_params(template_doc)

    if "Parameters" not in template_doc:
        template_doc['Parameters'] = []
    template_parameters = template_doc['Parameters']
    if (not "paramAmiName" in template_parameters):
        template_parameters['paramAmiName']    = collections.OrderedDict([("Description", "AMI Name"), ("Type", "String"), ("Default", "")])
    if (not "paramAmiCreated" in template_parameters):
        template_parameters['paramAmiCreated'] = collections.OrderedDict([("Description", "AMI Creation Date"), ("Type", "String"), ("Default", "")])

    json_template = aws_infra_util.json_save(template_doc)

    # save result

    print("** Final template:")
    print(json_template)
    print("")

    tmp = tempfile.NamedTemporaryFile(delete=False)
    tmp.write(json_template)
    tmp.close()

    # Load previous stack information to see if it has been deployed before

    describe_stack_command = [ 'aws', 'cloudformation', 'describe-stacks', '--stack-name', stack_name ]
    print("Checking for previous stack info: " + str(describe_stack_command))
    p = subprocess.Popen(describe_stack_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    if p.returncode:
        if (not output[1].endswith("does not exist\n")):
            sys.exit("Failed to retrieve old stack for " + stack_name + ": " + output[1])
        stack_oper = 'create-stack'
    else:
        stack_oper = 'update-stack'

    # Create/update stack

    params_doc = []
    for key in template_parameters.keys():
        if (key in os.environ):
            val = os.environ[key]
            print("Parameter " + key + ": using custom value " + val
            params_doc.append({ 'ParameterKey': key, 'ParameterValue': val })
        else:
            val = template_parameters[key]
            print("Parameter " + key + ": using default value " + val

    stack_command = \
        ['aws', 'cloudformation', stack_oper, '--stack-name',
         stack_name,
         '--template-body',
         'file://' + tmp.name,
         '--capabilities',
         'CAPABILITY_IAM',
         '--parameters',
         aws_infra_util.json_save(params_doc)
         ]

    currentTimeInCloudWatchFormat = datetime.datetime.now().strftime("%FT%H%%253A%M%%253A%SZ")

    print(stack_oper + ": " + str(stack_command))
    p = subprocess.Popen(stack_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    os.remove(tmp.name)
    if p.returncode:
        sys.exit(stack_oper + " failed: " + output[1])

    print(output[0])

    # Wait for update to complete

    cloudWatchNotice = "\nCloudWatch url:  https://console.aws.amazon.com/cloudwatch/home#logEvent:group=instanceDeployment;stream=" + stack_name + ";start=" + currentTimeInCloudWatchFormat + "\n"
    print(cloudWatchNotice)

    print("Waiting for update to complete:")
    while (True):
        p = subprocess.Popen(describe_stack_command,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                             env=dict(os.environ,
                                      AWS_ACCESS_KEY_ID=aws_access_key_id,
                                      AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                      AWS_SESSION_TOKEN=aws_session_token))
        output = p.communicate()
        if p.returncode:
            sys.exit("Describe stack failed: " + output[1])

        stack_info = aws_infra_util.json_load(output[0])
        status = stack_info['Stacks'][0]['StackStatus']
        print("Status: " + status)
        if (not status.endswith("_IN_PROGRESS")):
            break

        time.sleep(5)

    print(cloudWatchNotice)

    if (status != "UPDATE_COMPLETE"):
        sys.exit("Update stack failed: end state " + status)

    print("Done!")

if __name__ == '__main__':
    if len(sys.argv) < 4:
        sys.exit("Usage: deploy.py stack_name yaml_template\nParameters taken from environment as-is, missing parameters use defaults from template")
    deploy(sys.argv[1], sys.argv[2])
<commit_msg>cloudformation-update-stack: Bugfix<commit_after>#!/usr/bin/env python

# Copyright 2016 Nitor Creations Oy
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
import sys
import aws_infra_util
import os
import tempfile
import collections
import time
import datetime

def deploy(stack_name, yaml_template):

    # Disable buffering, from http://stackoverflow.com/questions/107705/disable-output-buffering
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    # Assume amibakery role

    p = subprocess.Popen(['aws', 'sts', 'assume-role', '--role-arn', 'arn:aws:iam::832585949989:role/amibakery', '--role-session-name', 'amibakery-deploy'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
    output = p.communicate()
    if p.returncode:
       sys.exit("Assume role failed: " + output[1])

    # AWS Credentials

    credentials = aws_infra_util.json_load(output[0])
    aws_access_key_id = credentials['Credentials']['AccessKeyId']
    aws_secret_access_key = credentials['Credentials']['SecretAccessKey']
    aws_session_token = credentials['Credentials']['SessionToken']

    # Get AMI metadata

    describe_ami_command = [ "aws", "ec2", "describe-images", "--image-ids", os.environ["paramAmi"] ]
    print("Checking AMI " + os.environ["paramAmi"] + " metadata: " + str(describe_ami_command))
    p = subprocess.Popen(describe_ami_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    if p.returncode:
        sys.exit("Failed to retrieve ami metadata for " + os.environm["paramAmi"])

    ami_meta = aws_infra_util.json_load(output[0])
    print("Result: " + aws_infra_util.json_save(ami_meta))
    os.environ["paramAmiName"] = ami_meta['Images'][0]['Name']
    os.environ["paramAmiCreated"] = ami_meta['Images'][0]['CreationDate']

    print("\n\n**** Deploying stack '" + stack_name + "' with template '" + yaml_template + "' and ami_id " + os.environ["paramAmi"])

    # Load yaml template and import scripts and patch userdata with metadata hash & params

    template_doc = aws_infra_util.yaml_load(open(yaml_template))
    template_doc = aws_infra_util.import_scripts(template_doc, yaml_template)
    aws_infra_util.patch_launchconf_userdata_with_metadata_hash_and_params(template_doc)

    if "Parameters" not in template_doc:
        template_doc['Parameters'] = []
    template_parameters = template_doc['Parameters']
    if (not "paramAmiName" in template_parameters):
        template_parameters['paramAmiName']    = collections.OrderedDict([("Description", "AMI Name"), ("Type", "String"), ("Default", "")])
    if (not "paramAmiCreated" in template_parameters):
        template_parameters['paramAmiCreated'] = collections.OrderedDict([("Description", "AMI Creation Date"), ("Type", "String"), ("Default", "")])

    json_template = aws_infra_util.json_save(template_doc)

    # save result

    print("** Final template:")
    print(json_template)
    print("")

    tmp = tempfile.NamedTemporaryFile(delete=False)
    tmp.write(json_template)
    tmp.close()

    # Load previous stack information to see if it has been deployed before

    describe_stack_command = [ 'aws', 'cloudformation', 'describe-stacks', '--stack-name', stack_name ]
    print("Checking for previous stack info: " + str(describe_stack_command))
    p = subprocess.Popen(describe_stack_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    if p.returncode:
        if (not output[1].endswith("does not exist\n")):
            sys.exit("Failed to retrieve old stack for " + stack_name + ": " + output[1])
        stack_oper = 'create-stack'
    else:
        stack_oper = 'update-stack'

    # Create/update stack

    params_doc = []
    for key in template_parameters.keys():
        if (key in os.environ):
            val = os.environ[key]
            print("Parameter " + key + ": using custom value " + val)
            params_doc.append({ 'ParameterKey': key, 'ParameterValue': val })
        else:
            val = template_parameters[key]
            print("Parameter " + key + ": using default value " + val

    stack_command = \
        ['aws', 'cloudformation', stack_oper, '--stack-name',
         stack_name,
         '--template-body',
         'file://' + tmp.name,
         '--capabilities',
         'CAPABILITY_IAM',
         '--parameters',
         aws_infra_util.json_save(params_doc)
         ]

    currentTimeInCloudWatchFormat = datetime.datetime.now().strftime("%FT%H%%253A%M%%253A%SZ")

    print(stack_oper + ": " + str(stack_command))
    p = subprocess.Popen(stack_command,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                         env=dict(os.environ,
                                  AWS_ACCESS_KEY_ID=aws_access_key_id,
                                  AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                  AWS_SESSION_TOKEN=aws_session_token))
    output = p.communicate()
    os.remove(tmp.name)
    if p.returncode:
        sys.exit(stack_oper + " failed: " + output[1])

    print(output[0])

    # Wait for update to complete

    cloudWatchNotice = "\nCloudWatch url:  https://console.aws.amazon.com/cloudwatch/home#logEvent:group=instanceDeployment;stream=" + stack_name + ";start=" + currentTimeInCloudWatchFormat + "\n"
    print(cloudWatchNotice)

    print("Waiting for update to complete:")
    while (True):
        p = subprocess.Popen(describe_stack_command,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
                             env=dict(os.environ,
                                      AWS_ACCESS_KEY_ID=aws_access_key_id,
                                      AWS_SECRET_ACCESS_KEY=aws_secret_access_key,
                                      AWS_SESSION_TOKEN=aws_session_token))
        output = p.communicate()
        if p.returncode:
            sys.exit("Describe stack failed: " + output[1])

        stack_info = aws_infra_util.json_load(output[0])
        status = stack_info['Stacks'][0]['StackStatus']
        print("Status: " + status)
        if (not status.endswith("_IN_PROGRESS")):
            break

        time.sleep(5)

    print(cloudWatchNotice)

    if (status != "UPDATE_COMPLETE"):
        sys.exit("Update stack failed: end state " + status)

    print("Done!")

if __name__ == '__main__':
    if len(sys.argv) < 4:
        sys.exit("Usage: deploy.py stack_name yaml_template\nParameters taken from environment as-is, missing parameters use defaults from template")
    deploy(sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>from datetime import datetime
from functools import partial

from django.conf import settings
from django.contrib.postgres.search import SearchVector
from django.core.exceptions import ObjectDoesNotExist

from common.util import print_progress, setInterval
from iati.models import (
    Activity,
    ActivitySearch,
    ContactInfoDepartment,
    ContactInfoPersonName,
    ContactInfoJobTitle,
    ContactInfoMailingAddress,
    CountryBudgetItem,
    BudgetItemDescription,
    ResultTitle,
    ResultDescription,
    Conditions
)
from iati.transaction.models import (
    TransactionDescription,
    TransactionProvider,
    TransactionReceiver
)

from iati_synchroniser.models import Publisher
from iati_organisation.models import Organisation, OrganisationName

# TODO: prefetches - 2016-01-07
def reindex_activity(activity):
    if hasattr(
        settings, 'FTS_ENABLED'
    ) and getattr(settings, 'FTS_ENABLED') is False:
        return

    try:
        activity_search = ActivitySearch.objects.get(activity=activity.id)
    except ObjectDoesNotExist:
        activity_search = ActivitySearch()

    # data prep
    title_text = []
    try:
        for narrative in activity.title.narratives.all():
            title_text.append(narrative.content)
    except ObjectDoesNotExist as e:
        pass

    description_text = []
    for description in activity.description_set.all():
        for narrative in description.narratives.all():
            description_text.append(narrative.content)

    reporting_org_text = []
    for reporting_org in activity.reporting_organisations.all():
        reporting_org_text.append(reporting_org.normalized_ref)
        for narrative in reporting_org.narratives.all():
            reporting_org_text.append(narrative.content)

    # The Publisher of the activity is a Reporting Organisation
    try:
        for narrative in activity.publisher.organisation.name.narratives.all():
            reporting_org_text.append(narrative.content)
    except (
        Publisher.DoesNotExist, Organisation.DoesNotExist,
        OrganisationName.DoesNotExist
    ):
        pass

    participating_org_text = []
    for participating_org in activity.participating_organisations.all():
        participating_org_text.append(participating_org.normalized_ref)
        for narrative in participating_org.narratives.all():
            participating_org_text.append(narrative.content)

    recipient_country_text = []
    for country in activity.recipient_country.all():
        recipient_country_text.append(country.code)
        recipient_country_text.append(country.name)

    recipient_region_text = []
    for region in activity.recipient_region.all():
        recipient_region_text.append(region.code)
        recipient_region_text.append(region.name)

    sector_text = []
    for sector in activity.sector.all():
        sector_text.append(sector.code)
        sector_text.append(sector.name)

    document_link_text = []
    for document_link in activity.documentlink_set.all():
        document_link_text.append(document_link.url)

        for category in document_link.categories.all():
            document_link_text.append(category.name)

        try:
            title = document_link.documentlinktitle
            for narrative in title.narratives.all():
                document_link_text.append(narrative.content)
        except ObjectDoesNotExist as e:
            pass

    other_identifier_text = []
    for other_identifier in activity.otheridentifier_set.all():
        other_identifier_text.append(other_identifier.owner_ref)
        for narrative in other_identifier.narratives.all():
            other_identifier_text.append(narrative.content)

    contact_info_text = []
    for contact_info in activity.contactinfo_set.all():
        # iati-activities/iati-activity/contact-info/organisation/narrative
        try:
            for narrative in contact_info.organisation.narratives.all():
                contact_info_text.append(narrative.content)
        except contact_info.DoesNotExist:
            pass

        # iati-activities/iati-activity/contact-info/department/narrative
        try:
            for narrative in contact_info.department.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoDepartment.DoesNotExist:
            pass

        # iati-activities/iati-activity/contact-info/person-name/narrative
        try:
            for narrative in contact_info.person_name.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoPersonName.DoesNotExist:
            pass

        # iati-activities/iati-activity/contact-info/job-title/narrative
        try:
            for narrative in contact_info.job_title.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoJobTitle.DoesNotExist:
            pass

        # iati-activities/iati-activity/contact-info/mailing-address/narrative
        try:
            for narrative in contact_info.mailing_address.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoMailingAddress.DoesNotExist:
            pass

    location_text = []
    for location in activity.location_set.all():
        if location.ref:
            location_text.append(location.ref)

    # iati-activities/iati-activity/country-budget-items/budget-item/description/narrative
    country_budget_items_text = []
    try:
        for budget_item in activity.country_budget_items.budgetitem_set.all():
            try:
                for narrative in budget_item.description.narratives.all():
                    country_budget_items_text.append(narrative.content)
            except BudgetItemDescription.DoesNotExist:
                pass
    except CountryBudgetItem.DoesNotExist:
        pass

    # iati-activities/iati-activity/policy-marker/narrative
    policy_marker_text = []
    for activity_policy_marker in activity.activitypolicymarker_set.all():
        for narrative in activity_policy_marker.narratives.all():
            policy_marker_text.append(narrative.content)

    transaction_text = []
    for transaction in activity.transaction_set.all():
        if transaction.ref:
            transaction_text.append(transaction.ref)

        # iati-activities/iati-activity/transaction/description/narrative
        try:
            for narrative in transaction.description.narratives.all():
                transaction_text.append(narrative.content)
        except TransactionDescription.DoesNotExist:
            pass

        # iati-activities/iati-activity/transaction/provider-org/
        try:
            transaction_text.append(transaction.provider_organisation.ref)
            for narrative in transaction.provider_organisation.narratives.all():
                transaction_text.append(narrative.content)
        except TransactionProvider.DoesNotExist:
            pass

        # iati-activities/iati-activity/transaction/receiver-org/
        try:
            transaction_text.append(transaction.receiver_organisation.ref)
            for narrative in transaction.receiver_organisation.narratives.all():  # NOQA: E501
                transaction_text.append(narrative.content)
        except TransactionReceiver.DoesNotExist:
            pass

    related_activity_text = []
    for related_activity in activity.relatedactivity_set.all():
        related_activity_text.append(related_activity.ref)

    # iati-activities/iati-activity/conditions
    conditions_text = []
    try:
        for condition in activity.conditions.condition_set.all():
            for narrative in condition.narratives.all():
                conditions_text.append(narrative.content)
    except Conditions.DoesNotExist:
        pass

    result_text = []
    for result in activity.result_set.all():
        # iati-activities/iati-activity/result/title/narrative
        try:
            for narrative in result.resulttitle.narratives.all():
                result_text.append(narrative.content)
        except ResultTitle.DoesNotExist:
            pass

        # iati-activities/iati-activity/result/description/narrative
        try:
            for narrative in result.resultdescription.narratives.all():
                result_text.append(narrative.content)
        except ResultDescription.DoesNotExist:
            pass

        for result_indicator in result.resultindicator_set.all():
            # iati-activities/iati-activity/result/indicator/title/narrative
            try:
                for narrative in \
                        result_indicator.resultindicatortitle.narratives.all():
                    result_text.append(narrative.content)
            except result_indicator.resultindicatortitle.DoesNotExist:
                pass

            # iati-activities/iati-activity/result/indicator/description/narrative
            try:
                for narrative in result_indicator.resultindicatordescription.narratives.all():  # NOQA: E501
                    result_text.append(narrative.content)
            except result_indicator.resultindicatordescription.DoesNotExist:
                pass

            for result_indicator_period in result_indicator.resultindicatorperiod_set.all():  # NOQA: E501
                # iati-activities/iati-activity/result/indicator/period/target/comment/narrative
                for target in result_indicator_period.targets.all():
                    for target_comment in target.resultindicatorperiodtargetcomment_set.all():  # NOQA: E501
                        for narrative in target_comment.narratives.all():
                            result_text.append(narrative.content)

                # iati-activities/iati-activity/result/indicator/period/actual/comment/narrative
                for actual in result_indicator_period.actuals.all():
                    for actual_comment in actual.resultindicatorperiodactualcomment_set.all():  # NOQA: E501
                        for narrative in actual_comment.narratives.all():
                            result_text.append(narrative.content)

    activity_search.activity = activity
    activity_search.iati_identifier = activity.iati_identifier
    activity_search.title = " ".join(title_text)
    activity_search.description = " ".join(description_text)
    activity_search.reporting_org = " ".join(reporting_org_text)
    activity_search.participating_org = " ".join(participating_org_text)
    activity_search.recipient_country = " ".join(recipient_country_text)
    activity_search.recipient_region = " ".join(recipient_region_text)
    activity_search.sector = " ".join(sector_text)
    activity_search.document_link = " ".join(document_link_text)
    activity_search.other_identifier = " ".join(other_identifier_text)
    activity_search.contact_info = " ".join(contact_info_text)
    activity_search.location = " ".join(location_text)
    activity_search.country_budget_items = " ".join(country_budget_items_text)
    activity_search.transaction = " ".join(transaction_text)
    activity_search.policy_marker = " ".join(policy_marker_text)
    activity_search.related_activity = " ".join(related_activity_text)
    activity_search.conditions = " ".join(conditions_text)
    activity_search.result = " ".join(result_text)

    _config = 'english'

    combined_vector = SearchVector(
        'iati_identifier',
        config=_config
    ) + SearchVector(
        'title',
        config=_config
    ) + SearchVector(
        'description',
        config=_config
    ) + SearchVector(
        'reporting_org',
        config=_config
    ) + SearchVector(
        'participating_org',
        config=_config
    ) + SearchVector(
        'recipient_country',
        config=_config
    ) + SearchVector(
        'recipient_region',
        config=_config
    ) + SearchVector(
        'sector',
        config=_config
    ) + SearchVector(
        'document_link',
        config=_config
    ) + SearchVector(
        'other_identifier',
        config=_config
    ) + SearchVector(
        'contact_info',
        config=_config
    ) + SearchVector(
        'location',
        config=_config
    ) + SearchVector(
        'country_budget_items',
        config=_config
    ) + SearchVector(
        'policy_marker',
        config=_config
    ) + SearchVector(
        'transaction',

        config=_config
    )+ SearchVector(
        'related_activity',
        config=_config
    ) + SearchVector(
        'conditions',
        config=_config
    ) + SearchVector(
        'result',
        config=_config
    )

    activity_search.last_reindexed = datetime.now()
    activity_search.save()
    ActivitySearch.objects.filter(id=activity_search.id).update(
        search_vector_text=combined_vector)

def reindex_all_activities():

    progress = {
        'offset': 0,
        'count': Activity.objects.all().count()
    }

    setInterval(partial(print_progress, progress), 10)

    for activity in Activity.objects.all().iterator():
        reindex_activity(activity)
        progress['offset'] += 1

def reindex_activity_by_source(dataset_id):
    activities = Activity.objects.all().filter(dataset__id=dataset_id)
    for activity in activities:
        reindex_activity(activity)
<commit_msg>Updated one to one parse<commit_after>from datetime import datetime
from functools import partial

from django.conf import settings
from django.contrib.postgres.search import SearchVector
from django.core.exceptions import ObjectDoesNotExist

from common.util import print_progress, setInterval
from iati.models import (
    Activity,
    ActivitySearch,
    ContactInfoDepartment,
    ContactInfoPersonName,
    ContactInfoJobTitle,
    ContactInfoMailingAddress,
    ContactInfoOrganisation,
    CountryBudgetItem,
    BudgetItemDescription,
    ResultTitle,
    ResultDescription,
    ResultIndicatorTitle,
    ResultIndicatorDescription,
    Conditions,
    DocumentLinkTitle
)
from iati.transaction.models import (
    TransactionDescription,
    TransactionProvider,
    TransactionReceiver
)

from iati_synchroniser.models import Publisher
from iati_organisation.models import Organisation, OrganisationName

# TODO: prefetches - 2016-01-07
def reindex_activity(activity):
    if hasattr(
        settings, 'FTS_ENABLED'
    ) and getattr(settings, 'FTS_ENABLED') is False:
        return

    try:
        activity_search = ActivitySearch.objects.get(activity=activity.id)
    except ObjectDoesNotExist:
        activity_search = ActivitySearch()

    # data prep
    title_text = []
    try:
        for narrative in activity.title.narratives.all():
            title_text.append(narrative.content)
    except ObjectDoesNotExist as e:
        pass

    description_text = []
    for description in activity.description_set.all():
        for narrative in description.narratives.all():
            description_text.append(narrative.content)

    reporting_org_text = []
    for reporting_org in activity.reporting_organisations.all():
        reporting_org_text.append(reporting_org.normalized_ref)
        for narrative in reporting_org.narratives.all():
            reporting_org_text.append(narrative.content)

    # The Publisher of the activity is a Reporting Organisation
    try:
        for narrative in activity.publisher.organisation.name.narratives.all():
            reporting_org_text.append(narrative.content)
    except (
        Publisher.DoesNotExist, Organisation.DoesNotExist,
        OrganisationName.DoesNotExist
    ):
        pass

    participating_org_text = []
    for participating_org in activity.participating_organisations.all():
        participating_org_text.append(participating_org.normalized_ref)
        for narrative in participating_org.narratives.all():
            participating_org_text.append(narrative.content)

    recipient_country_text = []
    for country in activity.recipient_country.all():
        recipient_country_text.append(country.code)
        recipient_country_text.append(country.name)

    recipient_region_text = []
    for region in activity.recipient_region.all():
        recipient_region_text.append(region.code)
        recipient_region_text.append(region.name)

    sector_text = []
    for sector in activity.sector.all():
        sector_text.append(sector.code)
        sector_text.append(sector.name)

    document_link_text = []
    for document_link in activity.documentlink_set.all():
        document_link_text.append(document_link.url)

        for category in document_link.categories.all():
            document_link_text.append(category.name)

        try:
            title = document_link.documentlinktitle
            for narrative in title.narratives.all():
                document_link_text.append(narrative.content)
        except DocumentLinkTitle.DoesNotExist as e:
            pass

    other_identifier_text = []
    for other_identifier in activity.otheridentifier_set.all():
        other_identifier_text.append(other_identifier.owner_ref)
        for narrative in other_identifier.narratives.all():
            other_identifier_text.append(narrative.content)

    contact_info_text = []
    for contact_info in activity.contactinfo_set.all():
        # iati-activities/iati-activity/contact-info/organisation/narrative
        try:
            organisation = ContactInfoOrganisation.objects.get(
                contact_info=contact_info
            )
            for narrative in organisation.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoOrganisation.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/contact-info/department/narrative
        try:
            for narrative in contact_info.department.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoDepartment.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/contact-info/person-name/narrative
        try:
            for narrative in contact_info.person_name.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoPersonName.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/contact-info/job-title/narrative
        try:
            for narrative in contact_info.job_title.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoJobTitle.DoesNotExist:
            pass

        # iati-activities/iati-activity/contact-info/mailing-address/narrative
        try:
            for narrative in contact_info.mailing_address.narratives.all():
                contact_info_text.append(narrative.content)
        except ContactInfoMailingAddress.DoesNotExist as e:
            pass

    location_text = []
    for location in activity.location_set.all():
        if location.ref:
            location_text.append(location.ref)

    # iati-activities/iati-activity/country-budget-items/budget-item/description/narrative
    country_budget_items_text = []
    try:
        for budget_item in activity.country_budget_items.budgetitem_set.all():
            try:
                for narrative in budget_item.description.narratives.all():
                    country_budget_items_text.append(narrative.content)
            except BudgetItemDescription.DoesNotExist as e:
                pass
    except CountryBudgetItem.DoesNotExist as e:
        pass

    # iati-activities/iati-activity/policy-marker/narrative
    policy_marker_text = []
    for activity_policy_marker in activity.activitypolicymarker_set.all():
        for narrative in activity_policy_marker.narratives.all():
            policy_marker_text.append(narrative.content)

    transaction_text = []
    for transaction in activity.transaction_set.all():
        if transaction.ref:
            transaction_text.append(transaction.ref)

        # iati-activities/iati-activity/transaction/description/narrative
        try:
            for narrative in transaction.description.narratives.all():
                transaction_text.append(narrative.content)
        except TransactionDescription.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/transaction/provider-org/
        try:
            transaction_text.append(transaction.provider_organisation.ref)
            for narrative in transaction.provider_organisation.narratives.all():
                transaction_text.append(narrative.content)
        except TransactionProvider.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/transaction/receiver-org/
        try:
            transaction_text.append(transaction.receiver_organisation.ref)
            for narrative in transaction.receiver_organisation.narratives.all():  # NOQA: E501
                transaction_text.append(narrative.content)
        except TransactionReceiver.DoesNotExist as e:
            pass

    related_activity_text = []
    for related_activity in activity.relatedactivity_set.all():
        related_activity_text.append(related_activity.ref)

    # iati-activities/iati-activity/conditions
    conditions_text = []
    try:
        for condition in activity.conditions.condition_set.all():
            for narrative in condition.narratives.all():
                conditions_text.append(narrative.content)
    except Conditions.DoesNotExist as e:
        pass

    result_text = []
    for result in activity.result_set.all():
        # iati-activities/iati-activity/result/title/narrative
        try:
            for narrative in result.resulttitle.narratives.all():
                result_text.append(narrative.content)
        except ResultTitle.DoesNotExist as e:
            pass

        # iati-activities/iati-activity/result/description/narrative
        try:
            for narrative in result.resultdescription.narratives.all():
                result_text.append(narrative.content)
        except ResultDescription.DoesNotExist as e:
            pass

        for result_indicator in result.resultindicator_set.all():
            # iati-activities/iati-activity/result/indicator/title/narrative
            try:
                for narrative in \
                        result_indicator.resultindicatortitle.narratives.all():
                    result_text.append(narrative.content)
            except ResultIndicatorTitle.DoesNotExist as e:
                pass

            # iati-activities/iati-activity/result/indicator/description/narrative
            try:
                for narrative in result_indicator.resultindicatordescription.narratives.all():  # NOQA: E501
                    result_text.append(narrative.content)
            except ResultIndicatorDescription.DoesNotExist as e:
                pass

            for result_indicator_period in result_indicator.resultindicatorperiod_set.all():  # NOQA: E501
                # iati-activities/iati-activity/result/indicator/period/target/comment/narrative
                for target in result_indicator_period.targets.all():
                    for target_comment in target.resultindicatorperiodtargetcomment_set.all():  # NOQA: E501
                        for narrative in target_comment.narratives.all():
                            result_text.append(narrative.content)

                # iati-activities/iati-activity/result/indicator/period/actual/comment/narrative
                for actual in result_indicator_period.actuals.all():
                    for actual_comment in actual.resultindicatorperiodactualcomment_set.all():  # NOQA: E501
                        for narrative in actual_comment.narratives.all():
                            result_text.append(narrative.content)

    activity_search.activity = activity
    activity_search.iati_identifier = activity.iati_identifier
    activity_search.title = " ".join(title_text)
    activity_search.description = " ".join(description_text)
    activity_search.reporting_org = " ".join(reporting_org_text)
    activity_search.participating_org = " ".join(participating_org_text)
    activity_search.recipient_country = " ".join(recipient_country_text)
    activity_search.recipient_region = " ".join(recipient_region_text)
    activity_search.sector = " ".join(sector_text)
    activity_search.document_link = " ".join(document_link_text)
    activity_search.other_identifier = " ".join(other_identifier_text)
    activity_search.contact_info = " ".join(contact_info_text)
    activity_search.location = " ".join(location_text)
    activity_search.country_budget_items = " ".join(country_budget_items_text)
    activity_search.transaction = " ".join(transaction_text)
    activity_search.policy_marker = " ".join(policy_marker_text)
    activity_search.related_activity = " ".join(related_activity_text)
    activity_search.conditions = " ".join(conditions_text)
    activity_search.result = " ".join(result_text)

    _config = 'english'

    combined_vector = SearchVector(
        'iati_identifier',
        config=_config
    ) + SearchVector(
        'title',
        config=_config
    ) + SearchVector(
        'description',
        config=_config
    ) + SearchVector(
        'reporting_org',
        config=_config
    ) + SearchVector(
        'participating_org',
        config=_config
    ) + SearchVector(
        'recipient_country',
        config=_config
    ) + SearchVector(
        'recipient_region',
        config=_config
    ) + SearchVector(
        'sector',
        config=_config
    ) + SearchVector(
        'document_link',
        config=_config
    ) + SearchVector(
        'other_identifier',
        config=_config
    ) + SearchVector(
        'contact_info',
        config=_config
    ) + SearchVector(
        'location',
        config=_config
    ) + SearchVector(
        'country_budget_items',
        config=_config
    ) + SearchVector(
        'policy_marker',
        config=_config
    ) + SearchVector(
        'transaction',

        config=_config
    )+ SearchVector(
        'related_activity',
        config=_config
    ) + SearchVector(
        'conditions',
        config=_config
    ) + SearchVector(
        'result',
        config=_config
    )

    activity_search.last_reindexed = datetime.now()
    activity_search.save()
    ActivitySearch.objects.filter(id=activity_search.id).update(
        search_vector_text=combined_vector)

def reindex_all_activities():

    progress = {
        'offset': 0,
        'count': Activity.objects.all().count()
    }

    setInterval(partial(print_progress, progress), 10)

    for activity in Activity.objects.all().iterator():
        reindex_activity(activity)
        progress['offset'] += 1

def reindex_activity_by_source(dataset_id):
    activities = Activity.objects.all().filter(dataset__id=dataset_id)
    for activity in activities:
        reindex_activity(activity)
<|endoftext|>


<commit_before>import os
import random
from functools import wraps

from flask import (
    redirect,
    render_template,
    request,
    Response,
    send_from_directory,
)
from werkzeug import secure_filename

from shrls import app
from shrls.models import (
    DBSession,
    Url,
    Snippet,
    allowed_shortner_chars,
)

def check_auth(username, password):
    """This function is called to check if a username /
    password combination is valid.
    """
    return all([username == app.config['shrls_username'],
                password == app.config['shrls_password']])

def authenticate():
    """Sends a 401 response that enables basic auth"""
    return Response(
        'Could not verify your access level for that URL.\n'
        'You have to login with proper credentials', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

def not_found():
    if app.config['shrls_redirect_unknown']:
        return redirect(app.config['shrls_redirect_url'], code=302)
    else:
        return Response("File not found", 404)

@app.route('/')
def index():
    return not_found()

@app.route('/code/<url_id>')
@app.route('/c/<url_id>')
def render_code_snippet(url_id):
    redirect_obj = DBSession.query(Snippet).filter(Snippet.alias == url_id).first()
    if redirect_obj:
        redirect_obj.views += 1
        DBSession.add(redirect_obj)
        DBSession.commit()
        return render_template('snippet.html', code=redirect_obj)
    return not_found()

@app.route('/uploads/<path:filename>')
@app.route('/u/<path:filename>')
def return_uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)

@app.route('/<path:url_id>')
def url_redirect(url_id):
    url_id = url_id.split('.')[0]
    redirect_obj = DBSession.query(Url).filter(Url.alias == url_id).first()
    if redirect_obj:
        location = redirect_obj.location
        redirect_obj.views += 1
        DBSession.add(redirect_obj)
        DBSession.commit()
        return redirect(location, code=302)
    return not_found()

@app.route('/admin/')
@requires_auth
def admin_index():
    urls = DBSession.query(Url).order_by(Url.created_at.asc()).all()
    return render_template('admin.html', urls=urls)

def create_url(longurl, shorturl=None, creator=None):
    shrl = Url(longurl)
    if shorturl:
        obj = DBSession.query(Url).filter(Url.alias == shorturl).first()
        if obj:
            DBSession.delete(obj)
            DBSession.commit()
        shrl.alias = shorturl
    if creator:
        shrl.alias = "{}/{}".format(creator, shrl.alias)
    DBSession.add(shrl)
    DBSession.commit()
    return '{}/{}'.format(app.config['shrls_base_url'], shrl.alias)

@app.route('/admin/create', methods=['GET'])
@requires_auth
def render_url():
    creator = request.args.get('c')
    longurl = request.args.get('u')
    shortid = request.args.get('s')
    url_only = request.args.get('url_only')
    if not(longurl):
        return "prompt('No url specified.')"
    alias = create_url(longurl, shorturl=shortid, creator=creator)
    if url_only:
        return alias
    else:
        return "prompt('The url has been shortened', '{}');".format(alias)

@app.route('/admin/snippet', methods=['POST'])
@requires_auth
def create_snippet():
    print request.args
    content = request.form.get('c')
    title = request.form.get('t')
    shortid = request.form.get('s')
    if not content:
        return "Error"
    shrl = Snippet(content, title=title)
    if shortid:
        obj = DBSession.query(Snippet).filter(Snippet.alias == shortid).first()
        if obj:
            DBSession.delete(obj)
            DBSession.commit()
        shrl.alias = shortid
    DBSession.add(shrl)
    DBSession.commit()
    alias = '{}/c/{}'.format(app.config['shrls_base_url'], shrl.alias)
    alias = create_url(alias)
    return alias

@app.route('/admin/upload', methods=['POST'])
@requires_auth
def upload_file():
    f = request.files['file']
    save_as = request.args.get('s')

    name = secure_filename(f.filename)
    extension = name.split('.')[-1]
    if not save_as:
        save_as = ''.join(
            [random.choice(allowed_shortner_chars) for _ in range(5)]
        )
    filename = "{}.{}".format(save_as, extension)
    f.save(os.path.join(
        app.config['UPLOAD_FOLDER'],
        filename)
    )
    alias = "{}/u/{}".format(app.config['shrls_base_url'], filename)
    alias = create_url(alias)
    return alias
<commit_msg>Revert url ordering problem<commit_after>import os
import random
from functools import wraps

from flask import (
    redirect,
    render_template,
    request,
    Response,
    send_from_directory,
)
from werkzeug import secure_filename

from shrls import app
from shrls.models import (
    DBSession,
    Url,
    Snippet,
    allowed_shortner_chars,
)

def check_auth(username, password):
    """This function is called to check if a username /
    password combination is valid.
    """
    return all([username == app.config['shrls_username'],
                password == app.config['shrls_password']])

def authenticate():
    """Sends a 401 response that enables basic auth"""
    return Response(
        'Could not verify your access level for that URL.\n'
        'You have to login with proper credentials', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

def not_found():
    if app.config['shrls_redirect_unknown']:
        return redirect(app.config['shrls_redirect_url'], code=302)
    else:
        return Response("File not found", 404)

@app.route('/')
def index():
    return not_found()

@app.route('/code/<url_id>')
@app.route('/c/<url_id>')
def render_code_snippet(url_id):
    redirect_obj = DBSession.query(Snippet).filter(Snippet.alias == url_id).first()
    if redirect_obj:
        redirect_obj.views += 1
        DBSession.add(redirect_obj)
        DBSession.commit()
        return render_template('snippet.html', code=redirect_obj)
    return not_found()

@app.route('/uploads/<path:filename>')
@app.route('/u/<path:filename>')
def return_uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)

@app.route('/<path:url_id>')
def url_redirect(url_id):
    url_id = url_id.split('.')[0]
    redirect_obj = DBSession.query(Url).filter(Url.alias == url_id).first()
    if redirect_obj:
        location = redirect_obj.location
        redirect_obj.views += 1
        DBSession.add(redirect_obj)
        DBSession.commit()
        return redirect(location, code=302)
    return not_found()

@app.route('/admin/')
@requires_auth
def admin_index():
    urls = DBSession.query(Url).order_by(Url.created_at.desc()).all()
    return render_template('admin.html', urls=urls)

def create_url(longurl, shorturl=None, creator=None):
    shrl = Url(longurl)
    if shorturl:
        obj = DBSession.query(Url).filter(Url.alias == shorturl).first()
        if obj:
            DBSession.delete(obj)
            DBSession.commit()
        shrl.alias = shorturl
    if creator:
        shrl.alias = "{}/{}".format(creator, shrl.alias)
    DBSession.add(shrl)
    DBSession.commit()
    return '{}/{}'.format(app.config['shrls_base_url'], shrl.alias)

@app.route('/admin/create', methods=['GET'])
@requires_auth
def render_url():
    creator = request.args.get('c')
    longurl = request.args.get('u')
    shortid = request.args.get('s')
    url_only = request.args.get('url_only')
    if not(longurl):
        return "prompt('No url specified.')"
    alias = create_url(longurl, shorturl=shortid, creator=creator)
    if url_only:
        return alias
    else:
        return "prompt('The url has been shortened', '{}');".format(alias)

@app.route('/admin/snippet', methods=['POST'])
@requires_auth
def create_snippet():
    print request.args
    content = request.form.get('c')
    title = request.form.get('t')
    shortid = request.form.get('s')
    if not content:
        return "Error"
    shrl = Snippet(content, title=title)
    if shortid:
        obj = DBSession.query(Snippet).filter(Snippet.alias == shortid).first()
        if obj:
            DBSession.delete(obj)
            DBSession.commit()
        shrl.alias = shortid
    DBSession.add(shrl)
    DBSession.commit()
    alias = '{}/c/{}'.format(app.config['shrls_base_url'], shrl.alias)
    alias = create_url(alias)
    return alias

@app.route('/admin/upload', methods=['POST'])
@requires_auth
def upload_file():
    f = request.files['file']
    save_as = request.args.get('s')

    name = secure_filename(f.filename)
    extension = name.split('.')[-1]
    if not save_as:
        save_as = ''.join(
            [random.choice(allowed_shortner_chars) for _ in range(5)]
        )
    filename = "{}.{}".format(save_as, extension)
    f.save(os.path.join(
        app.config['UPLOAD_FOLDER'],
        filename)
    )
    alias = "{}/u/{}".format(app.config['shrls_base_url'], filename)
    alias = create_url(alias)
    return alias
<|endoftext|>