<commit_before> #!/usr/bin/env python
import urllib
import urllib2
import json
import re
import os.path
import readline, glob
# import subprocess
import sys
import pprint
from mutagen.mp3 import MP3
from mutagen.id3 import ID3, APIC, TIT2, ID3, TALB, TPE1, TPE2, TRCK, TCON, TDRC, error

move_to_itunes = False

class tcolors:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    END = '\033[0m'
    BOLD = '\033[1m'

def validate_url(url):
    try:
        urllib2.urlopen(url)
        return True
    except:
        return False

def download_path():
    if move_to_itunes == True:
        return '~/Music/iTunes/iTunes Music/Automatically Add to iTunes/'
    else:
        return '~/Downloads/'

def change_song_details(audio_path, title, artist_info, track_num):
    mp3 = MP3(audio_path, ID3=ID3)
    set_tags(mp3)
    set_album_art(mp3, artist_info['album_art'], 'image/jpg')
    mp3.save()

    id3 = ID3(audio_path)
    set_album_name(id3, artist_info['album'])
    set_title(id3, title)
    set_artist(id3, artist_info['artist'])
    set_track_number(id3, str(track_num))
    # set_year(id3, artist_info['year'])
    # set_genre(id3, artist_info['genre'])
    id3.save()

def set_tags(mp3):
    try:
        mp3.add_tags()
    except error as e:
        # print e
        pass

def set_album_art(mp3, art_path, image_type):
    try:
        mp3.tags.add(
            APIC(
                encoding = 3, # 3 is for utf-8
                mime = image_type, # image/jpeg or image/png
                type = 3, # 3 is for the cover image
                desc = u'Cover',
                data = open(art_path).read()
            )
        )
    except error as e:
        print e
        pass

def set_album_name(id3, album_name):
    try:
        id3.add(TALB(encoding = 3, text = album_name))
    except error as e:
        print e
        pass

def set_title(id3, title):
    try:
        id3.add(TIT2(encoding = 3, text = title))
    except error as e:
        print e
        pass

def set_artist(id3, artist):
    try:
        id3.add(TPE2(encoding = 3, text = artist))
    except error as e:
        print e
        pass

def set_track_number(id3, track_number):
    try:
        id3.add(TRCK(encoding = 3, text = track_number))
    except error as e:
        print e
        pass

def set_year(id3, year):
    try:
        id3.add(TDRC(encoding = 3, text = year))
    except error as e:
        print e
        pass

def set_genre(id3, genre):
    try:
        id3.add(TCON(encoding = 3, text = genre))
    except error as e:
        print e
        pass

def make_directory(path):
    if not os.path.exists(os.path.expanduser(path)):
        try:
            os.makedirs(os.path.expanduser(path))
        except error as e:
            print e
            pass

def download_file(url, path, filename, album_json, title, track_num):
    make_directory(path)
    download_url = urllib2.urlopen(url)
    file_and_path = os.path.expanduser(path + filename)
    file = open(file_and_path, 'wb')
    meta = download_url.info()
    file_size = int(meta.getheaders('Content-Length')[0])

    print '\nDownloading: %s Size: %s' % (tcolors.BOLD + filename + tcolors.END, tcolors.BOLD + str(round(file_size / float(1000000), 2)) + "MB" + tcolors.END)

    file_size_dl = 0
    block_sz = 8192

    while True:
        buffer = download_url.read(block_sz)

        if not buffer:
            if track_num:
                change_song_details(file_and_path, title, album_json, track_num)
            else:
                album_json['album_art'] = file_and_path
            break

        file_size_dl += len(buffer)
        file.write(buffer)
        status = r'%10d  [%3.2f%%]' % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)

        print status,

    file.close()

def download_album(album_json):
    directory = download_path() + re.sub('/', '', album_json['artistinfo'][0]['album']) + '/'

    download_file(album_json['artistinfo'][0]['album_art'], directory, 'cover.jpg', album_json['artistinfo'][0], '', 0)

    for t in album_json['trackinfo']:
        download_file(t['file']['mp3-128'], directory, create_file_name(t['track_num'], t['title']), album_json['artistinfo'][0], t['title'], t['track_num'])

    return True

def create_file_name(track_number, title):
    return re.sub('/', '', ('0' if track_number < 10 else '') + str(track_number) + ' ' + title + '.mp3')

def find_album_json(garbage):
    album_json = None
    track_regex = re.compile('trackinfo\s\:\s\[')

    artist_name = sanitise_variable(re.compile('artist:\s\"'), garbage)
    album_name = sanitise_variable(re.compile('album_title:\s\"'), garbage)
    album_art_url = sanitise_variable(re.compile('artFullsizeUrl:\s\"'), garbage)
    track_json = re.sub('\}\]', '}]}', re.sub(track_regex, ',\"trackinfo\":[', find_json(garbage, track_regex, ',', ']')))

    if artist_name and album_name and track_json:
        extra_json = '{ "artistinfo": [{ "artist":"%s", "album":"%s", "album_art":"%s" }]' % (artist_name, album_name, album_art_url)
        album_json = json.loads(re.sub('\\\\', '', extra_json + track_json))

        print '\nDownloading: ' + tcolors.BLUE + album_name  + tcolors.END

    return album_json

def sanitise_variable(regex, garbage):
    return re.sub('"', '', re.sub(regex, '', find_json(garbage, regex, ',', '"')))

def find_json(garbage, regex, nail, coffin):
    for m in regex.finditer(garbage):
        start = m.start()

    if start:
        i = start
        end = None
        while not end:
            if garbage[i] is nail and garbage[i-1] is coffin:
                end = i
            i += 1

        return garbage[start:end]
    else:
        return None

def rip_it_up(album_url):
    response = urllib2.urlopen(album_url)

    if(response.getcode() == 200):
        garbage = response.read()
        album_json = find_album_json(garbage)
        if album_json:
            success = download_album(album_json)
    else:
        success = False

    return success

def check_argvs():
    for args in sys.argv:
        if args == '-i':
            move_to_itunes = True
            sys.argv[1] = sys.argv[2]

if len(sys.argv) > 1:
    check_argvs()

    album_url = sys.argv[1]

    if validate_url(album_url):
        if(rip_it_up(album_url)):
            print '\n\nDownloading: ' + tcolors.GREEN + 'Completed' + tcolors.END
        else:
            print tcolors.RED + '\n\nAh, that\'s a real ding there' + tcolors.END
    else:
        print 'What this is?'
else:
    print 'Gimmie something to rip bud!'

<commit_msg>Added some better validation<commit_after> #!/usr/bin/env python
import urllib
import urllib2
import json
import re
import os.path
import readline, glob
import string
import sys
import pprint
from urlparse import urlparse
from mutagen.mp3 import MP3
from mutagen.id3 import ID3, APIC, TIT2, ID3, TALB, TPE1, TPE2, TRCK, TCON, TDRC, error

move_to_itunes = False

class tcolors:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    END = '\033[0m'
    BOLD = '\033[1m'

def validate_url(url):
    url = urlparse(url)

    if url.scheme != 'https' and url.scheme !='http':
        print 'The url needs to start with http or https'
        return False

    domain = string.split(url.netloc, '.')

    if (domain[0] == 'www' and domain[0] != 'bandcamp') and (domain[0] != 'bandcamp'):
        print 'This doesn\'t look like a bandcamp url'
        return False

    domain = string.split(url.path, '/')

    if domain[1] != 'album':
        print 'You must provide an album'
        return False

    try:
        urllib2.urlopen(url.geturl())
        return True
    except:
        return False

def download_path():
    if move_to_itunes == True:
        return '~/Music/iTunes/iTunes Music/Automatically Add to iTunes/'
    else:
        return '~/Downloads/'

def change_song_details(audio_path, title, artist_info, track_num):
    mp3 = MP3(audio_path, ID3=ID3)
    set_tags(mp3)
    set_album_art(mp3, artist_info['album_art'], 'image/jpg')
    mp3.save()

    id3 = ID3(audio_path)
    set_album_name(id3, artist_info['album'])
    set_title(id3, title)
    set_artist(id3, artist_info['artist'])
    set_track_number(id3, str(track_num))
    # set_year(id3, artist_info['year'])
    # set_genre(id3, artist_info['genre'])
    id3.save()

def set_tags(mp3):
    try:
        mp3.add_tags()
    except error as e:
        # print e
        pass

def set_album_art(mp3, art_path, image_type):
    try:
        mp3.tags.add(
            APIC(
                encoding = 3, # 3 is for utf-8
                mime = image_type, # image/jpeg or image/png
                type = 3, # 3 is for the cover image
                desc = u'Cover',
                data = open(art_path).read()
            )
        )
    except error as e:
        print e
        pass

def set_album_name(id3, album_name):
    try:
        id3.add(TALB(encoding = 3, text = album_name))
    except error as e:
        print e
        pass

def set_title(id3, title):
    try:
        id3.add(TIT2(encoding = 3, text = title))
    except error as e:
        print e
        pass

def set_artist(id3, artist):
    try:
        id3.add(TPE2(encoding = 3, text = artist))
    except error as e:
        print e
        pass

def set_track_number(id3, track_number):
    try:
        id3.add(TRCK(encoding = 3, text = track_number))
    except error as e:
        print e
        pass

def set_year(id3, year):
    try:
        id3.add(TDRC(encoding = 3, text = year))
    except error as e:
        print e
        pass

def set_genre(id3, genre):
    try:
        id3.add(TCON(encoding = 3, text = genre))
    except error as e:
        print e
        pass

def make_directory(path):
    if not os.path.exists(os.path.expanduser(path)):
        try:
            os.makedirs(os.path.expanduser(path))
        except error as e:
            print e
            pass

def download_file(url, path, filename, album_json, title, track_num):
    make_directory(path)
    download_url = urllib2.urlopen(url)
    file_and_path = os.path.expanduser(path + filename)
    file = open(file_and_path, 'wb')
    meta = download_url.info()
    file_size = int(meta.getheaders('Content-Length')[0])

    print '\nDownloading: %s Size: %s' % (tcolors.BOLD + filename + tcolors.END, tcolors.BOLD + str(round(file_size / float(1000000), 2)) + "MB" + tcolors.END)

    file_size_dl = 0
    block_sz = 4096

    while True:
        buffer = download_url.read(block_sz)

        if not buffer:
            if track_num:
                change_song_details(file_and_path, title, album_json, track_num)
            else:
                album_json['album_art'] = file_and_path
            break

        file_size_dl += len(buffer)
        file.write(buffer)
        status = r'%10d  [%3.2f%%]' % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)

        print status,

    file.close()

def download_album(album_json):
    directory = download_path() + re.sub('/', '', album_json['artistinfo'][0]['album']) + '/'

    download_file(album_json['artistinfo'][0]['album_art'], directory, 'cover.jpg', album_json['artistinfo'][0], '', 0)

    for t in album_json['trackinfo']:
        download_file(t['file']['mp3-128'], directory, create_file_name(t['track_num'], t['title']), album_json['artistinfo'][0], t['title'], t['track_num'])

    return True

def create_file_name(track_number, title):
    return re.sub('/', '', ('0' if track_number < 10 else '') + str(track_number) + ' ' + title + '.mp3')

def find_album_json(garbage):
    album_json = None
    track_regex = re.compile('trackinfo\s\:\s\[')

    artist_name = sanitise_variable(re.compile('artist:\s\"'), garbage)
    album_name = sanitise_variable(re.compile('album_title:\s\"'), garbage)
    album_art_url = sanitise_variable(re.compile('artFullsizeUrl:\s\"'), garbage)
    track_json = re.sub('\}\]', '}]}', re.sub(track_regex, ',\"trackinfo\":[', find_json(garbage, track_regex, ',', ']')))

    if artist_name and album_name and track_json:
        extra_json = '{ "artistinfo": [{ "artist":"%s", "album":"%s", "album_art":"%s" }]' % (artist_name, album_name, album_art_url)
        album_json = json.loads(re.sub('\\\\', '', extra_json + track_json))

        print '\nDownloading: ' + tcolors.BLUE + album_name  + tcolors.END

    return album_json

def sanitise_variable(regex, garbage):
    return re.sub('"', '', re.sub(regex, '', find_json(garbage, regex, ',', '"')))

def find_json(garbage, regex, nail, coffin):
    for m in regex.finditer(garbage):
        start = m.start()

    if start:
        i = start
        end = None
        while not end:
            if garbage[i] is nail and garbage[i-1] is coffin:
                end = i
            i += 1

        return garbage[start:end]
    else:
        return None

def rip_it_up(album_url):
    response = urllib2.urlopen(album_url)

    if(response.getcode() == 200):
        garbage = response.read()
        album_json = find_album_json(garbage)
        if album_json:
            success = download_album(album_json)
    else:
        success = False

    return success

def check_argvs():
    for args in sys.argv:
        if args == '-i':
            move_to_itunes = True
            sys.argv[1] = sys.argv[2]

if len(sys.argv) > 1:
    check_argvs()

    album_url = sys.argv[1]

    if validate_url(album_url):
        if(rip_it_up(album_url)):
            print '\n\nDownloading: ' + tcolors.GREEN + 'Completed' + tcolors.END
        else:
            print tcolors.RED + '\n\nAh, that\'s a real ding there' + tcolors.END
    else:
        print 'What this is?'
else:
    print 'Gimmie something to rip bud!'

<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, MessageResult, Result, QueuedSeries
from functools import reduce

from django.db import connection
from django.db.models import Q

from django.contrib.postgres.search import SearchQuery, SearchVector, SearchVectorField
from django.db.models import Lookup
from django.db.models.fields import Field

@Field.register_lookup
class NotEqual(Lookup):
    lookup_name = "ne"

    def as_sql(self, compiler, connection):
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params = lhs_params + rhs_params
        return "%s <> %s" % (lhs, rhs), params

class InvalidSearchTerm(Exception):
    pass

# Hack alert: Django wraps each argument to to_tsvector with a COALESCE function,
# and that causes postgres not to use the index.  Monkeypatch the constructor
# to skip that step, which we do not need since the subject field is not nullable.
class NonNullSearchVector(SearchVector):
    function = "to_tsvector"
    arg_joiner = " || ' ' || "
    _output_field = SearchVectorField()
    config = None

    def __init__(self, *expressions, **extra):
        super(SearchVector, self).__init__(*expressions, **extra)
        self.config = self.extra.get("config", self.config)
        self.weight = None

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete or is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

"not:X" is the opposite of "is:X". "+X" and "-X" are shorter synonyms of "is:X"
and "not:X" respectively.

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Search by maintainer associated with the changeset

 - Syntax: maintained-by:NAME
 - Syntax: maint:NAME

NAME can be the name, email or a substring of MAINTAINERS file entries of the
maintainer.

---

### Search by result

Syntax:

 - pending:NAME, failure:NAME, running:NAME - any result with the given
   name is in the pending/failure/running state
 - success:NAME - all results with the given name are in the success state
   (and there is at least one result with the given name)

where NAME can be e.g. "git", "testing", "testing.TEST-NAME"

Example:

    success:git
    failure:testing.FreeBSD

---

### Search by review state

Syntax:

 - accept:USERNAME or ack:USERNAME - the series was marked as accepted by the user
 - reject:USERNAME or nack:USERNAME - the series was marked as reject by the user
 - review:USERNAME - the series was marked as accepted or rejected by the user
 - watch:USERNAME - the series is in the user's watched queue

USERNAME can be "me" to identify the current user

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""

    def _make_filter_subquery(self, model, q):
        message_ids = model.objects.filter(q).values("message_id")
        return Q(id__in=message_ids)

    def _make_filter_result(self, term, **kwargs):
        q = Q(name=term, **kwargs) | Q(name__startswith=term + ".", **kwargs)
        return self._make_filter_subquery(MessageResult, q)

    def _make_filter_age(self, cond):
        import datetime

        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or unit not in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q

    def _add_to_keywords(self, t):
        self._last_keywords.append(t)
        return Q()

    def _make_filter_is(self, cond):
        if cond == "complete":
            return Q(is_complete=True)
        elif cond == "pull":
            self._add_to_keywords("PULL")
            return Q(subject__contains="[PULL") | Q(subject__contains="[GIT PULL")
        elif cond == "reviewed":
            return Q(is_reviewed=True)
        elif cond in ("obsoleted", "old"):
            return Q(is_obsolete=True)
        elif cond == "applied":
            return self._make_filter_subquery(
                MessageResult, Q(name="git", status=Result.SUCCESS)
            )
        elif cond == "tested":
            return Q(is_tested=True)
        elif cond == "merged":
            return Q(is_merged=True)
        return None

    def _make_filter_queue(self, username, user, **kwargs):
        if username == "me":
            if not user.is_authenticated:
                # Django hack to return an always false Q object
                return Q(pk=None)
            q = Q(user=user, **kwargs)
        else:
            q = Q(user__username=username, **kwargs)
        return self._make_filter_subquery(QueuedSeries, q)

    def _make_filter(self, term, user):
        if term.startswith("age:"):
            cond = term[term.find(":") + 1 :]
            return self._make_filter_age(cond)
        elif term[0] in "<>" and len(term) > 1:
            return self._make_filter_age(term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1 :]
            return Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1 :]
            return Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1 :]
            return self._add_to_keywords(cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1 :]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            return Q(message_id=cond)
        elif term.startswith("is:"):
            return self._make_filter_is(term[3:]) or self._add_to_keywords(term)
        elif term.startswith("not:"):
            return ~self._make_filter_is(term[4:]) or self._add_to_keywords(term)
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1 :]
            if cond == "replies":
                return Q(last_comment_date__isnull=False)
            else:
                return Q(properties__name=cond)
        elif term.startswith("failure:"):
            return self._make_filter_result(term[8:], status=Result.FAILURE)
        elif term.startswith("success:"):
            # What we want is "all results are successes", but the only way to
            # express it is "there is a result and not (any result is not a success)".
            return self._make_filter_result(term[8:]) & ~self._make_filter_result(
                term[8:], status__ne=Result.SUCCESS
            )
        elif term.startswith("pending:"):
            return self._make_filter_result(term[8:], status=Result.PENDING)
        elif term.startswith("running:"):
            return self._make_filter_result(term[8:], status=Result.RUNNING)
        elif (
            term.startswith("ack:")
            or term.startswith("accept:")
            or term.startswith("accepted:")
        ):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="accept")
        elif (
            term.startswith("nack:")
            or term.startswith("reject:")
            or term.startswith("rejected:")
        ):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="reject")
        elif term.startswith("review:") or term.startswith("reviewed:"):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(
                username, user, name__in=["accept", "reject"]
            )
        elif term.startswith("watch:") or term.startswith("watched:"):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="watched")
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1 :]
            self._projects.add(cond)
            return Q(project__name=cond) | Q(project__parent_project__name=cond)
        elif term.startswith("maintained-by:") or term.startswith("maint:"):
            cond = term[term.find(":") + 1 :]
            if cond == "me" and user:
                cond = user.email
            return Q(maintainers__icontains=cond)

        # Keyword in subject is the default
        return self._add_to_keywords(term)

    def _process_term(self, term, user):
        """ Return a Q object that will be applied to the query """
        is_plusminus = neg = False
        if term[0] in "+-!":
            neg = term[0] != "+"
            is_plusminus = term[0] != "!"
            term = term[1:]

        if is_plusminus and ":" not in term:
            q = self._make_filter_is(term) or self._add_to_keywords(term)
        else:
            q = self._make_filter(term, user)
        if neg:
            return ~q
        else:
            return q

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, user=None, queryset=None):
        self._last_keywords = []
        self._projects = set()
        q = reduce(
            lambda x, y: x & y, map(lambda t: self._process_term(t, user), terms), Q()
        )
        if queryset is None:
            queryset = Message.objects.series_heads()
        if self._last_keywords:
            if connection.vendor == "postgresql":
                queryset = queryset.annotate(
                    subjsearch=NonNullSearchVector("subject", config="english")
                )
                searchq = reduce(
                    lambda x, y: x & y,
                    map(
                        lambda x: SearchQuery(x, config="english"), self._last_keywords
                    ),
                )
                q = q & Q(subjsearch=searchq)
            else:
                q = reduce(
                    lambda x, y: x & Q(subject__icontains=y), self._last_keywords, q
                )

        return queryset.filter(q)

    def query_test_message(self, query, message):
        queryset = Message.objects.filter(id=message.id)
        terms = [x.strip() for x in query.split() if x.strip()]
        return self.search_series(*terms, queryset=queryset).first()
<commit_msg>add is:obsolete as synonym of is:obsoleted<commit_after>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from .models import Message, MessageResult, Result, QueuedSeries
from functools import reduce

from django.db import connection
from django.db.models import Q

from django.contrib.postgres.search import SearchQuery, SearchVector, SearchVectorField
from django.db.models import Lookup
from django.db.models.fields import Field

@Field.register_lookup
class NotEqual(Lookup):
    lookup_name = "ne"

    def as_sql(self, compiler, connection):
        lhs, lhs_params = self.process_lhs(compiler, connection)
        rhs, rhs_params = self.process_rhs(compiler, connection)
        params = lhs_params + rhs_params
        return "%s <> %s" % (lhs, rhs), params

class InvalidSearchTerm(Exception):
    pass

# Hack alert: Django wraps each argument to to_tsvector with a COALESCE function,
# and that causes postgres not to use the index.  Monkeypatch the constructor
# to skip that step, which we do not need since the subject field is not nullable.
class NonNullSearchVector(SearchVector):
    function = "to_tsvector"
    arg_joiner = " || ' ' || "
    _output_field = SearchVectorField()
    config = None

    def __init__(self, *expressions, **extra):
        super(SearchVector, self).__init__(*expressions, **extra)
        self.config = self.extra.get("config", self.config)
        self.weight = None

class SearchEngine(object):
    """

The general form of search string is a list of terms separated with space:

    QUERY = TERM TERM ...

Each term can be either a plain keyword, or a predict in the form of
`PRED:EXP`, where PRED is the predefined filter and EXP is the parameters to be
applied to the filter. As a simple example:

    bugfix from:Bob to:George age:>1w

to search emails titled as 'bugfix' (a subject keyword filter) from Bob (a
sender filter) to George (a recipient filter) before 1 week ago (an age
filter).

or

    bugfix from:Bob is:reviewed not:obsoleted

to search all emails from Bob that have "bugfix" in subject, and have been
reviewed but is not obsoleted (by a new revision of this series). Because there
are syntax shortcut for some predicts, it can be simplified as:

    from:Bob fix +reviewed -tested

---

## Supported filter types

### Search by age

 - Syntax: age:AGE
 - Syntax: >AGE
 - Syntax: <AGE

Filter by age of the message. Supports "d" (day), "w" (week), "m" (month) and "y" (year) as units. Examples:

 - age:1d
 - age:>2d
 - age:<1w
 - <1m
 - \>1w

---

### Search by series state

Syntax:

 - is:reviewed - all the patches in the series is reviewed
 - is:obsolete, is:obsoleted, is:old - the series has newer version
 - is:complete - the series has all the patches it contains
 - is:merged - the series is included in the project's git tree
 - is:pull - the series is a pull request
 - is:applied - a git tree is available for the series
 - has:replies - the series received a reply (apart from patches sent by the submitter)

Example:

    is:reviewed

"not:X" is the opposite of "is:X". "+X" and "-X" are shorter synonyms of "is:X"
and "not:X" respectively.

---

### Search addresses

 - Syntax: from:ADDRESS
 - Syntax: to:ADDRESS

Compare the address info of message. Example:

    from:alice to:bob

---

### Search by maintainer associated with the changeset

 - Syntax: maintained-by:NAME
 - Syntax: maint:NAME

NAME can be the name, email or a substring of MAINTAINERS file entries of the
maintainer.

---

### Search by result

Syntax:

 - pending:NAME, failure:NAME, running:NAME - any result with the given
   name is in the pending/failure/running state
 - success:NAME - all results with the given name are in the success state
   (and there is at least one result with the given name)

where NAME can be e.g. "git", "testing", "testing.TEST-NAME"

Example:

    success:git
    failure:testing.FreeBSD

---

### Search by review state

Syntax:

 - accept:USERNAME or ack:USERNAME - the series was marked as accepted by the user
 - reject:USERNAME or nack:USERNAME - the series was marked as reject by the user
 - review:USERNAME - the series was marked as accepted or rejected by the user
 - watch:USERNAME - the series is in the user's watched queue

USERNAME can be "me" to identify the current user

---

### Reverse condition

 - Syntax: !TERM

Negative of an expression. Example:

    !is:reviewed     (query series that are not reviewed)
    !has:replies     (query series that have not received any comment)

---

### Search by message id

 - Syntax: id:MESSAGE-ID

Exact match of message-id. Example:

    id:<1416902879-17422-1-git-send-email-user@domain.com>

or

    id:1416902879-17422-1-git-send-email-user@domain.com

---

### Search by text

 - Syntax: KEYWORD

Search text keyword in the email message. Example:

    regression

"""

    def _make_filter_subquery(self, model, q):
        message_ids = model.objects.filter(q).values("message_id")
        return Q(id__in=message_ids)

    def _make_filter_result(self, term, **kwargs):
        q = Q(name=term, **kwargs) | Q(name__startswith=term + ".", **kwargs)
        return self._make_filter_subquery(MessageResult, q)

    def _make_filter_age(self, cond):
        import datetime

        def human_to_seconds(n, unit):
            if unit == "d":
                return n * 86400
            elif unit == "w":
                return n * 86400 * 7
            elif unit == "m":
                return n * 86400 * 30
            elif unit == "y":
                return n * 86400 * 365
            raise Exception("No unit specified")

        if cond.startswith("<"):
            less = True
            cond = cond[1:]
        elif cond.startswith(">"):
            less = False
            cond = cond[1:]
        else:
            less = False
        num, unit = cond[:-1], cond[-1].lower()
        if not num.isdigit() or unit not in "dwmy":
            raise InvalidSearchTerm("Invalid age string: %s" % cond)
        sec = human_to_seconds(int(num), unit)
        p = datetime.datetime.now() - datetime.timedelta(0, sec)
        if less:
            q = Q(date__gte=p)
        else:
            q = Q(date__lte=p)
        return q

    def _add_to_keywords(self, t):
        self._last_keywords.append(t)
        return Q()

    def _make_filter_is(self, cond):
        if cond == "complete":
            return Q(is_complete=True)
        elif cond == "pull":
            self._add_to_keywords("PULL")
            return Q(subject__contains="[PULL") | Q(subject__contains="[GIT PULL")
        elif cond == "reviewed":
            return Q(is_reviewed=True)
        elif cond in ("obsoleted", "old", "obsolete"):
            return Q(is_obsolete=True)
        elif cond == "applied":
            return self._make_filter_subquery(
                MessageResult, Q(name="git", status=Result.SUCCESS)
            )
        elif cond == "tested":
            return Q(is_tested=True)
        elif cond == "merged":
            return Q(is_merged=True)
        return None

    def _make_filter_queue(self, username, user, **kwargs):
        if username == "me":
            if not user.is_authenticated:
                # Django hack to return an always false Q object
                return Q(pk=None)
            q = Q(user=user, **kwargs)
        else:
            q = Q(user__username=username, **kwargs)
        return self._make_filter_subquery(QueuedSeries, q)

    def _make_filter(self, term, user):
        if term.startswith("age:"):
            cond = term[term.find(":") + 1 :]
            return self._make_filter_age(cond)
        elif term[0] in "<>" and len(term) > 1:
            return self._make_filter_age(term)
        elif term.startswith("from:"):
            cond = term[term.find(":") + 1 :]
            return Q(sender__icontains=cond)
        elif term.startswith("to:"):
            cond = term[term.find(":") + 1 :]
            return Q(recipients__icontains=cond)
        elif term.startswith("subject:"):
            cond = term[term.find(":") + 1 :]
            return self._add_to_keywords(cond)
        elif term.startswith("id:"):
            cond = term[term.find(":") + 1 :]
            if cond[0] == "<" and cond[-1] == ">":
                cond = cond[1:-1]
            return Q(message_id=cond)
        elif term.startswith("is:"):
            return self._make_filter_is(term[3:]) or self._add_to_keywords(term)
        elif term.startswith("not:"):
            return ~self._make_filter_is(term[4:]) or self._add_to_keywords(term)
        elif term.startswith("has:"):
            cond = term[term.find(":") + 1 :]
            if cond == "replies":
                return Q(last_comment_date__isnull=False)
            else:
                return Q(properties__name=cond)
        elif term.startswith("failure:"):
            return self._make_filter_result(term[8:], status=Result.FAILURE)
        elif term.startswith("success:"):
            # What we want is "all results are successes", but the only way to
            # express it is "there is a result and not (any result is not a success)".
            return self._make_filter_result(term[8:]) & ~self._make_filter_result(
                term[8:], status__ne=Result.SUCCESS
            )
        elif term.startswith("pending:"):
            return self._make_filter_result(term[8:], status=Result.PENDING)
        elif term.startswith("running:"):
            return self._make_filter_result(term[8:], status=Result.RUNNING)
        elif (
            term.startswith("ack:")
            or term.startswith("accept:")
            or term.startswith("accepted:")
        ):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="accept")
        elif (
            term.startswith("nack:")
            or term.startswith("reject:")
            or term.startswith("rejected:")
        ):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="reject")
        elif term.startswith("review:") or term.startswith("reviewed:"):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(
                username, user, name__in=["accept", "reject"]
            )
        elif term.startswith("watch:") or term.startswith("watched:"):
            username = term[term.find(":") + 1 :]
            return self._make_filter_queue(username, user, name="watched")
        elif term.startswith("project:"):
            cond = term[term.find(":") + 1 :]
            self._projects.add(cond)
            return Q(project__name=cond) | Q(project__parent_project__name=cond)
        elif term.startswith("maintained-by:") or term.startswith("maint:"):
            cond = term[term.find(":") + 1 :]
            if cond == "me" and user:
                cond = user.email
            return Q(maintainers__icontains=cond)

        # Keyword in subject is the default
        return self._add_to_keywords(term)

    def _process_term(self, term, user):
        """ Return a Q object that will be applied to the query """
        is_plusminus = neg = False
        if term[0] in "+-!":
            neg = term[0] != "+"
            is_plusminus = term[0] != "!"
            term = term[1:]

        if is_plusminus and ":" not in term:
            q = self._make_filter_is(term) or self._add_to_keywords(term)
        else:
            q = self._make_filter(term, user)
        if neg:
            return ~q
        else:
            return q

    def last_keywords(self):
        return getattr(self, "_last_keywords", [])

    def project(self):
        return next(iter(self._projects)) if len(self._projects) == 1 else None

    def search_series(self, *terms, user=None, queryset=None):
        self._last_keywords = []
        self._projects = set()
        q = reduce(
            lambda x, y: x & y, map(lambda t: self._process_term(t, user), terms), Q()
        )
        if queryset is None:
            queryset = Message.objects.series_heads()
        if self._last_keywords:
            if connection.vendor == "postgresql":
                queryset = queryset.annotate(
                    subjsearch=NonNullSearchVector("subject", config="english")
                )
                searchq = reduce(
                    lambda x, y: x & y,
                    map(
                        lambda x: SearchQuery(x, config="english"), self._last_keywords
                    ),
                )
                q = q & Q(subjsearch=searchq)
            else:
                q = reduce(
                    lambda x, y: x & Q(subject__icontains=y), self._last_keywords, q
                )

        return queryset.filter(q)

    def query_test_message(self, query, message):
        queryset = Message.objects.filter(id=message.id)
        terms = [x.strip() for x in query.split() if x.strip()]
        return self.search_series(*terms, queryset=queryset).first()
<|endoftext|>


<commit_before>#!/bin/python3

from collections import deque

class CastleOnGrid:

    CELL_BLOCKED_TOKEN = 'X'

    def __init__(self, grid, grid_size, start, goal):
        self.grid = grid
        self.grid_size = grid_size
        self.start = start
        self.goal = goal

    def min_moves(self):
        to_visit = deque()
        visited = []

        to_visit.append((self.start, 0))
        visited.append(self.start)

        current_coords = None
        while current_coords != self.goal:
            current_coords, current_depth = to_visit.popleft()
            for move in self.possible_moves(current_coords):
                if move not in visited:
                    to_visit.append((move, current_depth + 1))
                    visited.append(move)

        return current_depth

    def possible_moves(self, coords):
        possible = []

        row, col = coords
        for row_i in range(row + 1, self.grid_size):
            if self.grid[row_i][col] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                possible.append((row_i, col))

        for row_i in range(row - 1, -1, -1):
            if self.grid[row_i][col] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                possible.append((row_i, col))

        for col_i in range(col + 1, self.grid_size):
            if self.grid[row][col_i] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                possible.append((row, col_i))

        for col_i in range(col - 1, -1, -1):
            if self.grid[row][col_i] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                possible.append((row, col_i))

        return possible

if __name__ == '__main__':
    grid_size = int(input())
    grid = []

    for _ in range(grid_size):
        grid.append(list(input()))

    coords = list(map(int, input().split()))
    start = (coords[0], coords[1])
    goal = (coords[2], coords[3])

    castle_on_grid = CastleOnGrid(grid, grid_size, start, goal)
    # print(castle_on_grid.possible_moves((1, 1)))
    min_moves = castle_on_grid.min_moves()
    print(min_moves)
<commit_msg>Try using possible_moves generator<commit_after>#!/bin/python3

from collections import deque

class CastleOnGrid:

    CELL_BLOCKED_TOKEN = 'X'

    def __init__(self, grid, grid_size, start, goal):
        self.grid = grid
        self.grid_size = grid_size
        self.start = start
        self.goal = goal

    def min_moves(self):
        to_visit = deque()
        visited = []

        to_visit.append((self.start, 0))
        visited.append(self.start)

        current_coords = None
        while current_coords != self.goal:
            current_coords, current_depth = to_visit.popleft()
            for move in self.possible_moves(current_coords):
                if move not in visited:
                    to_visit.append((move, current_depth + 1))
                    visited.append(move)

        return current_depth

    def possible_moves(self, coords):
        row, col = coords
        for row_i in range(row + 1, self.grid_size):
            if self.grid[row_i][col] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                yield (row_i, col)

        for row_i in range(row - 1, -1, -1):
            if self.grid[row_i][col] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                yield (row_i, col)

        for col_i in range(col + 1, self.grid_size):
            if self.grid[row][col_i] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                yield (row, col_i)

        for col_i in range(col - 1, -1, -1):
            if self.grid[row][col_i] == self.CELL_BLOCKED_TOKEN:
                break
            else:
                yield (row, col_i)

if __name__ == '__main__':
    grid_size = int(input())
    grid = []

    for _ in range(grid_size):
        grid.append(list(input()))

    coords = list(map(int, input().split()))
    start = (coords[0], coords[1])
    goal = (coords[2], coords[3])

    castle_on_grid = CastleOnGrid(grid, grid_size, start, goal)
    # print(castle_on_grid.possible_moves((1, 1)))
    min_moves = castle_on_grid.min_moves()
    print(min_moves)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 Google and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tokenization class for model PEGASUS."""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging

if is_sentencepiece_available():
    from .tokenization_pegasus import PegasusTokenizer
else:
    PegasusTokenizer = None

logger = logging.get_logger(__name__)

SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/pegasus-xsum": 512,
}

class PegasusTokenizerFast(PreTrainedTokenizerFast):
    r"""
    Construct a "fast" PEGASUS tokenizer (backed by HuggingFace's `tokenizers` library). Based on `Unigram
    <https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models>`__.

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the main methods.
    Users should refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (:obj:`str`):
            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
            The end of sequence token.

            .. note::

                When building a sequence using special tokens, this is not the token that is used for the end of
                sequence. The token used is the :obj:`sep_token`.
        unk_token (:obj:`str`, `optional`, defaults to :obj:`"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        mask_token (:obj:`str`, `optional`, defaults to :obj:`"<mask_2>"`):
            The token used for masking single token values. This is the token used when training this model with masked
            language modeling (MLM). This is the token that the PEGASUS encoder will try to predict during pretraining.
            It corresponds to `[MASK2]` in `PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive
            Summarization <https://arxiv.org/pdf/1912.08777.pdf>`__.
        mask_token_sent (:obj:`str`, `optional`, defaults to :obj:`"<mask_1>"`):
            The token used for masking whole target sentences. This is the token used when training this model with gap
            sentences generation (GSG). This is the sentence that the PEGASUS decoder will try to predict during
            pretraining. It corresponds to `[MASK1]` in `PEGASUS: Pre-training with Extracted Gap-sentences for
            Abstractive Summarization <https://arxiv.org/pdf/1912.08777.pdf>`__.
        additional_special_tokens (:obj:`List[str]`, `optional`):
            Additional special tokens used by the tokenizer. If no additional_special_tokens are provided <mask_2> and
            <unk_2, ..., unk_102> are used as additional special tokens corresponding to the `original PEGASUS
            tokenizer
            <https://github.com/google-research/pegasus/blob/939830367bcf411193d2b5eca2f2f90f3f9260ca/pegasus/ops/pretrain_parsing_ops.cc#L66>`__
            that uses the tokens 2 - 104 only for pretraining
    """
    offset = 103  # entries 2-104 are only used for pretraining
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = PegasusTokenizer
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        tokenizer_file=None,
        pad_token="<pad>",
        eos_token="</s>",
        unk_token="<unk>",
        mask_token="<mask_2>",
        mask_token_sent="<mask_1>",
        additional_special_tokens=None,
        **kwargs
    ):
        if additional_special_tokens is not None:
            assert isinstance(
                additional_special_tokens, list
            ), f"additional_special_tokens should be of type {type(list)}, but is {type(additional_special_tokens)}"

            additional_special_tokens_extended = (
                ([mask_token_sent] + additional_special_tokens)
                if mask_token_sent not in additional_special_tokens
                else additional_special_tokens
            )
            # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
            additional_special_tokens_extended += [
                f"<unk_{i}>" for i in range(len(additional_special_tokens_extended), self.offset - 1)
            ]

            if len(set(additional_special_tokens_extended)) != len(additional_special_tokens_extended):
                raise ValueError(
                    f"Please make sure that the provided additional_special_tokens do not contain an incorrectly shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."
                )
            additional_special_tokens = additional_special_tokens_extended
        else:
            additional_special_tokens = [mask_token_sent]
            additional_special_tokens += [f"<unk_{i}>" for i in range(2, self.offset)]

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            pad_token=pad_token,
            eos_token=eos_token,
            unk_token=unk_token,
            mask_token=mask_token,
            mask_token_sent=mask_token_sent,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file

    def _special_token_mask(self, seq):
        all_special_ids = set(self.all_special_ids)  # call it once instead of inside list comp
        all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

        assert all_special_ids == set(
            range(len(self.additional_special_tokens) + 3)
        ), f"There should be 3 special tokens: mask_token, pad_token, and eos_token + {len(self.additional_special_tokens)} additional_special_tokens, but got {all_special_ids}"

        return [1 if x in all_special_ids else 0 for x in seq]

    def get_special_tokens_mask(
        self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """Get list where entries are [1] if a token is [eos] or [pad] else 0."""
        if already_has_special_tokens:
            return self._special_token_mask(token_ids_0)
        elif token_ids_1 is None:
            return self._special_token_mask(token_ids_0) + [1]
        else:
            return self._special_token_mask(token_ids_0 + token_ids_1) + [1]

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        """
        Build model inputs from a sequence by adding eos to the end. no bos token is added to the front.

        - single sequence: ``X </s>``
        - pair of sequences: ``A B </s>`` (not intended use)

        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs to which the special tokens will be added
            token_ids_1 (:obj:`List[int]`, `optional`):
                Optional second list of IDs for sequence pairs.

        Returns:
            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return token_ids_0 + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_0 + token_ids_1 + [self.eos_token_id]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)
<commit_msg>fix styling<commit_after># coding=utf-8
# Copyright 2020 Google and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tokenization class for model PEGASUS."""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging

if is_sentencepiece_available():
    from .tokenization_pegasus import PegasusTokenizer
else:
    PegasusTokenizer = None

logger = logging.get_logger(__name__)

SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/pegasus-xsum": 512,
}

class PegasusTokenizerFast(PreTrainedTokenizerFast):
    r"""
    Construct a "fast" PEGASUS tokenizer (backed by HuggingFace's `tokenizers` library). Based on `Unigram
    <https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models>`__.

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the main
    methods. Users should refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (:obj:`str`):
            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
            The end of sequence token.

            .. note::

                When building a sequence using special tokens, this is not the token that is used for the end of
                sequence. The token used is the :obj:`sep_token`.
        unk_token (:obj:`str`, `optional`, defaults to :obj:`"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        mask_token (:obj:`str`, `optional`, defaults to :obj:`"<mask_2>"`):
            The token used for masking single token values. This is the token used when training this model with masked
            language modeling (MLM). This is the token that the PEGASUS encoder will try to predict during pretraining.
            It corresponds to `[MASK2]` in `PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive
            Summarization <https://arxiv.org/pdf/1912.08777.pdf>`__.
        mask_token_sent (:obj:`str`, `optional`, defaults to :obj:`"<mask_1>"`):
            The token used for masking whole target sentences. This is the token used when training this model with gap
            sentences generation (GSG). This is the sentence that the PEGASUS decoder will try to predict during
            pretraining. It corresponds to `[MASK1]` in `PEGASUS: Pre-training with Extracted Gap-sentences for
            Abstractive Summarization <https://arxiv.org/pdf/1912.08777.pdf>`__.
        additional_special_tokens (:obj:`List[str]`, `optional`):
            Additional special tokens used by the tokenizer. If no additional_special_tokens are provided <mask_2> and
            <unk_2, ..., unk_102> are used as additional special tokens corresponding to the `original PEGASUS
            tokenizer
            <https://github.com/google-research/pegasus/blob/939830367bcf411193d2b5eca2f2f90f3f9260ca/pegasus/ops/pretrain_parsing_ops.cc#L66>`__
            that uses the tokens 2 - 104 only for pretraining
    """
    offset = 103  # entries 2-104 are only used for pretraining
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = PegasusTokenizer
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        tokenizer_file=None,
        pad_token="<pad>",
        eos_token="</s>",
        unk_token="<unk>",
        mask_token="<mask_2>",
        mask_token_sent="<mask_1>",
        additional_special_tokens=None,
        **kwargs
    ):
        if additional_special_tokens is not None:
            assert isinstance(
                additional_special_tokens, list
            ), f"additional_special_tokens should be of type {type(list)}, but is {type(additional_special_tokens)}"

            additional_special_tokens_extended = (
                ([mask_token_sent] + additional_special_tokens)
                if mask_token_sent not in additional_special_tokens
                else additional_special_tokens
            )
            # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
            additional_special_tokens_extended += [
                f"<unk_{i}>" for i in range(len(additional_special_tokens_extended), self.offset - 1)
            ]

            if len(set(additional_special_tokens_extended)) != len(additional_special_tokens_extended):
                raise ValueError(
                    f"Please make sure that the provided additional_special_tokens do not contain an incorrectly shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."
                )
            additional_special_tokens = additional_special_tokens_extended
        else:
            additional_special_tokens = [mask_token_sent]
            additional_special_tokens += [f"<unk_{i}>" for i in range(2, self.offset)]

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            pad_token=pad_token,
            eos_token=eos_token,
            unk_token=unk_token,
            mask_token=mask_token,
            mask_token_sent=mask_token_sent,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file

    def _special_token_mask(self, seq):
        all_special_ids = set(self.all_special_ids)  # call it once instead of inside list comp
        all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

        assert all_special_ids == set(
            range(len(self.additional_special_tokens) + 3)
        ), f"There should be 3 special tokens: mask_token, pad_token, and eos_token + {len(self.additional_special_tokens)} additional_special_tokens, but got {all_special_ids}"

        return [1 if x in all_special_ids else 0 for x in seq]

    def get_special_tokens_mask(
        self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """Get list where entries are [1] if a token is [eos] or [pad] else 0."""
        if already_has_special_tokens:
            return self._special_token_mask(token_ids_0)
        elif token_ids_1 is None:
            return self._special_token_mask(token_ids_0) + [1]
        else:
            return self._special_token_mask(token_ids_0 + token_ids_1) + [1]

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        """
        Build model inputs from a sequence by adding eos to the end. no bos token is added to the front.

        - single sequence: ``X </s>``
        - pair of sequences: ``A B </s>`` (not intended use)

        Args:
            token_ids_0 (:obj:`List[int]`):
                List of IDs to which the special tokens will be added
            token_ids_1 (:obj:`List[int]`, `optional`):
                Optional second list of IDs for sequence pairs.

        Returns:
            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return token_ids_0 + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_0 + token_ids_1 + [self.eos_token_id]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)
<|endoftext|>


<commit_before>import collections

class Delegate:
    def __init__(self, name):
        self.name = name

    def __get__(self, instance, owner):
        return getattr(instance, self.name)

class ArrayCache(list):
    # implicitly called magic methods don't invoke __getattribute__
    # https://docs.python.org/3/reference/datamodel.html#special-method-lookup
    # all method lookups obey the descriptor protocol
    # this is how the implicit api is defined in ccxt under the hood
    __iter__ = Delegate('__iter__')
    __setitem__ = Delegate('__setitem__')
    __delitem__ = Delegate('__delitem__')
    __len__ = Delegate('__len__')
    __contains__ = Delegate('__contains__')
    __reversed__ = Delegate('__reversed__')

    def __init__(self, max_size=None):
        super(list, self).__init__()
        self._deque = collections.deque([], max_size)

    def __eq__(self, other):
        return list(self) == other

    def __getattribute__(self, item):
        if item == 'append':
            return object.__getattribute__(self, item)
        deque = super(list, self).__getattribute__('_deque')
        return getattr(deque, item)

    def __repr__(self):
        return str(list(self))

    def __add__(self, other):
        return list(self) + other

    def __getitem__(self, item):
        deque = super(list, self).__getattribute__('_deque')
        if isinstance(item, slice):
            start, stop, step = item.indices(len(deque))
            return [deque[i] for i in range(start, stop, step)]
        else:
            return deque[item]

    def append(self, item):
        deque = super(list, self).__getattribute__('_deque')
        deque.append(item)

class ArrayCacheBySymbolById(ArrayCache):

    def append(self, item):
        deque = super(list, self).__getattribute__('_deque')
        found_indices = [i for i, v in enumerate(deque) if (v['symbol'] == item['symbol']) and (v['id'] == item['id'])]
        if len(found_indices):
            first_match = found_indices[0]
            deque[first_match] = item
        else:
            deque.append(item)
<commit_msg>cache.py edits<commit_after>import collections

class Delegate:
    def __init__(self, name):
        self.name = name

    def __get__(self, instance, owner):
        return getattr(instance, self.name)

class ArrayCache(list):
    # implicitly called magic methods don't invoke __getattribute__
    # https://docs.python.org/3/reference/datamodel.html#special-method-lookup
    # all method lookups obey the descriptor protocol
    # this is how the implicit api is defined in ccxt under the hood
    __iter__ = Delegate('__iter__')
    __setitem__ = Delegate('__setitem__')
    __delitem__ = Delegate('__delitem__')
    __len__ = Delegate('__len__')
    __contains__ = Delegate('__contains__')
    __reversed__ = Delegate('__reversed__')

    def __init__(self, max_size=None):
        super(list, self).__init__()
        self._deque = collections.deque([], max_size)

    def __eq__(self, other):
        return list(self) == other

    def __getattribute__(self, item):
        if item == 'append':
            return object.__getattribute__(self, item)
        deque = super(list, self).__getattribute__('_deque')
        return getattr(deque, item)

    def __repr__(self):
        return str(list(self))

    def __add__(self, other):
        return list(self) + other

    def __getitem__(self, item):
        deque = super(list, self).__getattribute__('_deque')
        if isinstance(item, slice):
            start, stop, step = item.indices(len(deque))
            return [deque[i] for i in range(start, stop, step)]
        else:
            return deque[item]

    def append(self, item):
        deque = super(list, self).__getattribute__('_deque')
        deque.append(item)

class ArrayCacheBySymbolById(ArrayCache):

    @staticmethod
    def match(a, b):
        return (a['symbol'] == b['symbol']) and (a['id'] == b['id'])

    def append(self, item):
        deque = super(list, self).__getattribute__('_deque')
        found_indices = [i for i, v in enumerate(deque) if ArrayCacheBySymbolById.match(v, item)]
        if len(found_indices):
            first_match = found_indices[0]
            deque[first_match] = item
        else:
            deque.append(item)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import flask_migrate
import flask_script

from seabus.common.database import db
from seabus.api.api import create_app as create_api_app

api_app = create_api_app('Dev')

api_manager = flask_script.Manager(api_app)

@api_manager.command
def apidev():
    api_app.run(
        host='0.0.0.0',
        port=6000,
        debug=True,
        use_reloader=True,
    )

if __name__ == '__main__':
    api_manager.run()
<commit_msg>[api] add prod command to api_manage<commit_after>#!/usr/bin/env python

import flask_migrate
import flask_script

from seabus.common.database import db
from seabus.api.api import create_app as create_api_app

api_app = create_api_app('Dev')

api_manager = flask_script.Manager(api_app)

@api_manager.command
def apidev():
    api_app.run(
        host='0.0.0.0',
        port=6000,
        debug=True,
        use_reloader=True,
    )

@api_manager.command
def apiprod():
    api_app.config.from_object('seabus.api.config.Prod')
    socketio.run(
        api_app,
        debug=False,
        use_reloader=False,
    )

if __name__ == '__main__':
    api_manager.run()
<|endoftext|>


<commit_before>from __future__ import absolute_import

from __future__ import unicode_literals

from custom.icds_reports.queries import get_cas_data_blob_file

import io
import os
import csv342 as csv

from custom.icds_reports.tests import OUTPUT_PATH, CSVTestCase
from six.moves import zip

class TestLocationView(CSVTestCase):
    always_include_columns = {'awc_id', 'case_id'}
    # indicator numbers:
    # 1 - child_health_monthly
    # 2 - ccs_record_monthly
    # 3 - agg_awc

    def _get_data_from_blobdb(self, indicator, state_id, month):
        sync, _ = get_cas_data_blob_file(indicator, state_id, month)
        with sync.get_file_from_blobdb() as fileobj:
            csv_file = io.TextIOWrapper(fileobj, encoding='utf-8')
            csv_data = list(csv.reader(csv_file))
        headers = csv_data[0]
        rows = csv_data[1:]
        for row_count, row in enumerate(rows):
            rows[row_count] = dict(zip(headers, row))
        return rows

    def test_child_health_monthly_cas_data(self):
        indicator = 1
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'child_health_monthly-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x
            ),
            sorted(
                csv_data,
                key=lambda x: x
            )
        )

    def test_ccs_record_monthly_cas_data(self):
        indicator = 2
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'ccs_record_monthly-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x
            ),
            sorted(
                csv_data,
                key=lambda x: x
            )
        )

    def test_agg_awc_cas_data(self):
        indicator = 3
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'agg_awc-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x
            ),
            sorted(
                csv_data,
                key=lambda x: x
            )
        )
<commit_msg>sort dicts by key<commit_after>from __future__ import absolute_import

from __future__ import unicode_literals

from custom.icds_reports.queries import get_cas_data_blob_file

import io
import os
import csv342 as csv

from custom.icds_reports.tests import OUTPUT_PATH, CSVTestCase
from six.moves import zip

class TestLocationView(CSVTestCase):
    always_include_columns = {'awc_id', 'case_id'}
    # indicator numbers:
    # 1 - child_health_monthly
    # 2 - ccs_record_monthly
    # 3 - agg_awc

    def _get_data_from_blobdb(self, indicator, state_id, month):
        sync, _ = get_cas_data_blob_file(indicator, state_id, month)
        with sync.get_file_from_blobdb() as fileobj:
            csv_file = io.TextIOWrapper(fileobj, encoding='utf-8')
            csv_data = list(csv.reader(csv_file))
        headers = csv_data[0]
        rows = csv_data[1:]
        for row_count, row in enumerate(rows):
            rows[row_count] = dict(zip(headers, row))
        return rows

    def test_child_health_monthly_cas_data(self):
        indicator = 1
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'child_health_monthly-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x['case_id']
            ),
            sorted(
                csv_data,
                key=lambda x: x['case_id']
            )
        )

    def test_ccs_record_monthly_cas_data(self):
        indicator = 2
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'ccs_record_monthly-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x['case_id']
            ),
            sorted(
                csv_data,
                key=lambda x: x['case_id']
            )
        )

    def test_agg_awc_cas_data(self):
        indicator = 3
        state_id = 'st1'
        month = '2017-05-01'
        csv_data = self._get_data_from_blobdb(indicator, state_id, month)
        expected_csv_file = self._load_csv(
            os.path.join(OUTPUT_PATH, 'agg_awc-st1-2017-05-01.csv')
        )
        self._fasterAssertListEqual(
            sorted(
                expected_csv_file,
                key=lambda x: x['awc_id']
            ),
            sorted(
                csv_data,
                key=lambda x: x['awc_id']
            )
        )
<|endoftext|>


<commit_before>from cwpoliticl.extensions.base_parser import BaseParser
from cwpoliticl.items import CacheItem, WDPost

class News18Parser(BaseParser):
    def __init__(self):
        from cwpoliticl.scraped_websites import WebsiteTypes
        self.url_from = WebsiteTypes.news18.value
        super(News18Parser, self).__init__()

    def parse_paginate(self, url, hxs, cache_db, history_db):
        select_block = '//*[@class="author-lest-blog cflip"]/*[@class="author-list flip-container"]'
        self._parse_block_for_pagination(url, hxs, cache_db, history_db, select_block)

    def _parse_block_for_pagination(self, url, hxs, cache_db, history_db, select_block):
        links = hxs.xpath(select_block).extract()

        for idx, link in enumerate(links):
            href_selector = '{}[{}]/div/div/figure/a/@href'.format(select_block, (idx + 1))
            thumbnail_selector = '{}[{}]/div/div/figure/a/img/@src'.format(select_block, (idx + 1))

            href = self.get_value_with_urljoin(hxs, href_selector, url)
            # If the link already exist on the history database, ignore it.
            if history_db.check_history_exist(href):
                continue

            thumbnail_src = self.get_value_response(hxs, thumbnail_selector)

            cache_db.save_cache(CacheItem.get_default(url=href, thumbnail_url=thumbnail_src, url_from=self.url_from))

    def parse(self, url, hxs, wd_rpc, thumbnail_url, access_denied_cookie):
        title = self.get_value_response(hxs, '//*[@class="section-blog-left-aricle"]/h1/text()')
        image_src = self.get_value_response(hxs,
                                            '//*[@class="section-blog-left-aricle"]/*[@class="articleimg"]/img/@src')

        content = self.get_all_value_response(hxs,
                                              '//*[@class="section-blog-left-aricle"]/*[@class="article_body"]/p/text()')

        # not found any tags on the detailed page.
        tags = []

        item = WDPost.get_default(url, self.url_from, title, image_src, thumbnail_url, content, tags,
                                  access_denied_cookie=access_denied_cookie)

        post_id = wd_rpc.post_to_wd(item)

        return item
<commit_msg>cwpoliticl.<commit_after>from cwpoliticl.extensions.base_parser import BaseParser
from cwpoliticl.items import CacheItem, WDPost

class News18Parser(BaseParser):
    def __init__(self):
        from cwpoliticl.scraped_websites import WebsiteTypes
        self.url_from = WebsiteTypes.news18.value
        super(News18Parser, self).__init__()

    def parse_paginate(self, url, hxs, cache_db, history_db):
        select_block = '//*[@class="author-lest-blog cflip"]/*[@class="author-list flip-container"]'
        self._parse_block_for_pagination(url, hxs, cache_db, history_db, select_block)

    def _parse_block_for_pagination(self, url, hxs, cache_db, history_db, select_block):
        links = hxs.xpath(select_block).extract()

        for idx, link in enumerate(links):
            href_selector = '{}[{}]/div/div/figure/a/@href'.format(select_block, (idx + 1))
            thumbnail_selector = '{}[{}]/div/div/figure/a/img/@src'.format(select_block, (idx + 1))

            href = self.get_value_with_urljoin(hxs, href_selector, url)
            # If the link already exist on the history database, ignore it.
            if history_db.check_history_exist(href):
                continue

            thumbnail_src = self.get_value_response(hxs, thumbnail_selector)

            cache_db.save_cache(CacheItem.get_default(url=href, thumbnail_url=thumbnail_src, url_from=self.url_from))

    def parse(self, url, hxs, wd_rpc, thumbnail_url, access_denied_cookie):
        title = self.get_value_response(hxs, '//*[@class="section-blog-left-aricle"]/h1/text()')
        image_src = self.get_value_response(hxs,
                                            '//*[@class="section-blog-left-aricle"]/*[@class="articleimg"]/img/@src')

        content = self.get_all_value_response(hxs,
                                              '//*[@class="section-blog-left-aricle"]/*[@id="article_body"]/p/text()')

        # not found any tags on the detailed page.
        tags = []

        item = WDPost.get_default(url, self.url_from, title, image_src, thumbnail_url, content, tags,
                                  access_denied_cookie=access_denied_cookie)

        post_id = wd_rpc.post_to_wd(item)

        return item
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pwd

from st2common import log as logging
from st2common.models.system.action import RemoteAction
from st2common.util.shell import quote_unix

__all__ = [
    'ParamikoRemoteCommandAction',
]

LOG = logging.getLogger(__name__)

LOGGED_USER_USERNAME = pwd.getpwuid(os.getuid())[0]

class ParamikoRemoteCommandAction(RemoteAction):

    def get_full_command_string(self):
        # Note: We pass -E to sudo because we want to preserve user provided
        # environment variables
        env_str = self._get_env_vars_export_string()
        cwd = self.get_cwd()

        if self.sudo:
            if env_str:
                command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command))
            else:
                command = quote_unix('cd %s && %s' % (cwd, self.command))

            command = 'sudo -E -- bash -c %s' % (command)
        else:
            if self.user and self.user != LOGGED_USER_USERNAME:
                # Need to use sudo to run as a different user
                user = quote_unix(self.user)
                if env_str:
                    command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command))
                else:
                    command = quote_unix('cd %s && %s' % (cwd, self.command))
                command = 'sudo -E -u %s -- bash -c %s' % (user, command)
            else:
                if env_str:
                    command = '%s && cd %s && %s' % (env_str, cwd,
                                                     self.command)
                else:
                    command = 'cd %s && %s' % (cwd, self.command)

        LOG.debug('Command to run on remote host will be: %s', command)
        return command
<commit_msg>Run remote command without sudo even if remote user is actually different<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pwd

from st2common import log as logging
from st2common.models.system.action import RemoteAction
from st2common.util.shell import quote_unix

__all__ = [
    'ParamikoRemoteCommandAction',
]

LOG = logging.getLogger(__name__)

LOGGED_USER_USERNAME = pwd.getpwuid(os.getuid())[0]

class ParamikoRemoteCommandAction(RemoteAction):

    def get_full_command_string(self):
        # Note: We pass -E to sudo because we want to preserve user provided
        # environment variables
        env_str = self._get_env_vars_export_string()
        cwd = self.get_cwd()

        if self.sudo:
            if env_str:
                command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command))
            else:
                command = quote_unix('cd %s && %s' % (cwd, self.command))

            command = 'sudo -E -- bash -c %s' % (command)
        else:
            if env_str:
                command = '%s && cd %s && %s' % (env_str, cwd,
                                                 self.command)
            else:
                command = 'cd %s && %s' % (cwd, self.command)

        LOG.debug('Command to run on remote host will be: %s', command)
        return command
<|endoftext|>