<commit_before>import logging; _L = logging.getLogger('openaddr.ci.work')

from .. import compat, util
from ..jobs import JOB_TIMEOUT

import os, json, tempfile, shutil, base64
from urllib.parse import urlparse, urljoin

MAGIC_OK_MESSAGE = 'Everything is fine'

def upload_file(s3, keyname, filename):
    ''' Create a new S3 key with filename contents, return its URL and MD5 hash.
    '''
    key = s3.new_key(keyname)

    kwargs = dict(policy='public-read', reduced_redundancy=True)
    key.set_contents_from_filename(filename, **kwargs)
    url = util.s3_key_url(key)

    return url, key.md5.decode('ascii')

def make_source_filename(source_name):
    '''
    '''
    return source_name.replace(u'/', u'--') + '.txt'

def assemble_output(s3, input, source_name, run_id, index_dirname):
    ''' Convert worker index dictionary to result output.

        The return of this function is passed through a few queues,
        and eventually is expected to be acceptable to RunState().
    '''
    output = {k: v for (k, v) in input.items()}

    if input['cache']:
        # e.g. /runs/0/cache.zip
        cache_path = os.path.join(index_dirname, input['cache'])
        key_name = '/runs/{run}/{cache}'.format(run=run_id, **input)
        url, fingerprint = upload_file(s3, key_name, cache_path)
        output['cache'], output['fingerprint'] = url, fingerprint

    if input['sample']:
        # e.g. /runs/0/sample.json
        sample_path = os.path.join(index_dirname, input['sample'])
        key_name = '/runs/{run}/{sample}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, sample_path)
        output['sample'] = url

    if input['processed']:
        # e.g. /runs/0/fr/paris.zip
        processed_path = os.path.join(index_dirname, input['processed'])
        package_args = input.get('website') or 'Unknown', input.get('license') or 'Unknown'
        archive_path = util.package_output(source_name, processed_path, *package_args)
        key_name = u'/runs/{run}/{name}.zip'.format(run=run_id, name=source_name)
        url, hash = upload_file(s3, key_name, archive_path)
        output['processed'], output['process hash'] = url, hash

        if os.path.exists(archive_path):
            os.remove(archive_path)

    if input['output']:
        # e.g. /runs/0/output.txt
        output_path = os.path.join(index_dirname, input['output'])
        key_name = '/runs/{run}/{output}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, output_path)
        output['output'] = url

    if input['preview']:
        # e.g. /runs/0/preview.png
        preview_path = os.path.join(index_dirname, input['preview'])
        key_name = '/runs/{run}/{preview}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, preview_path)
        output['preview'] = url

    return output

def do_work(s3, run_id, source_name, job_contents_b64, render_preview, output_dir, mapzen_key=None):
    ''' Do the actual work of running a source file in job_contents.
    '''
    _L.info('Doing work on source {}'.format(repr(source_name)))

    # Make a directory to run the whole job
    workdir = tempfile.mkdtemp(prefix='work-', dir=output_dir)

    # Write the user input to a file
    out_fn = os.path.join(workdir, make_source_filename(source_name))
    with open(out_fn, 'wb') as out_fp:
        out_fp.write(base64.b64decode(job_contents_b64))

    # Make a directory in which to run openaddr
    oa_dir = os.path.join(workdir, 'out')
    os.mkdir(oa_dir)

    # Invoke the job to do
    logfile_path = os.path.join(workdir, 'logfile.txt')
    cmd = 'openaddr-process-one', '-l', logfile_path, out_fn, oa_dir

    if render_preview and mapzen_key:
        cmd += ('--render-preview', '--mapzen-key', mapzen_key)
    else:
        cmd += ('--skip-preview', )

    try:
        known_error, cmd_status = False, 0
        timeout_seconds = JOB_TIMEOUT.seconds + JOB_TIMEOUT.days * 86400
        with open('/dev/null', 'a') as devnull:
            result_stdout = compat.check_output(cmd, timeout=timeout_seconds, stderr=devnull)
    except compat.TimeoutExpired as e:
        known_error, cmd_status, result_stdout = True, None, e.output
    except compat.CalledProcessError as e:
        known_error, cmd_status, result_stdout = True, e.returncode, e.output
    except Exception:
        known_error, cmd_status, result_stdout = False, None, None
        raise
    finally:
        if hasattr(result_stdout, 'decode'):
            # "The actual encoding of the output data may depend on the command
            # being invoked" - https://docs.python.org/3/library/subprocess.html
            result_stdout = result_stdout.decode('utf8', 'replace')

        if known_error:
            # Something went wrong; throw back an error result.
            key_name = '/runs/{run}/logfile.txt'.format(run=run_id)
            try:
                url, _ = upload_file(s3, key_name, logfile_path)
            except IOError:
                output = dict()
            else:
                output = dict(output=url)

            return dict(result_code=cmd_status, result_stdout=result_stdout,
                        message='Something went wrong in {0}'.format(*cmd),
                        output=output)

    result = dict(result_code=0, result_stdout=result_stdout,
                  message=MAGIC_OK_MESSAGE)

    # openaddr-process-one prints a path to index.json
    state_fullpath = result_stdout.strip()

    with open(state_fullpath) as file:
        index = dict(zip(*json.load(file)))

        for key in ('processed', 'sample', 'cache'):
            if not index[key] and not index.get('skipped'):
                result.update(result_code=-1, message='Failed to produce {} data'.format(key))

        index_dirname = os.path.dirname(state_fullpath)
        result['output'] = assemble_output(s3, index, source_name, run_id, index_dirname)

    shutil.rmtree(workdir)
    return result
<commit_msg>Ensured that run slippymap.mbtiles gets uploaded<commit_after>import logging; _L = logging.getLogger('openaddr.ci.work')

from .. import compat, util
from ..jobs import JOB_TIMEOUT

import os, json, tempfile, shutil, base64
from urllib.parse import urlparse, urljoin

MAGIC_OK_MESSAGE = 'Everything is fine'

def upload_file(s3, keyname, filename):
    ''' Create a new S3 key with filename contents, return its URL and MD5 hash.
    '''
    key = s3.new_key(keyname)

    kwargs = dict(policy='public-read', reduced_redundancy=True)
    key.set_contents_from_filename(filename, **kwargs)
    url = util.s3_key_url(key)

    return url, key.md5.decode('ascii')

def make_source_filename(source_name):
    '''
    '''
    return source_name.replace(u'/', u'--') + '.txt'

def assemble_output(s3, input, source_name, run_id, index_dirname):
    ''' Convert worker index dictionary to result output.

        The return of this function is passed through a few queues,
        and eventually is expected to be acceptable to RunState().
    '''
    output = {k: v for (k, v) in input.items()}

    if input['cache']:
        # e.g. /runs/0/cache.zip
        cache_path = os.path.join(index_dirname, input['cache'])
        key_name = '/runs/{run}/{cache}'.format(run=run_id, **input)
        url, fingerprint = upload_file(s3, key_name, cache_path)
        output['cache'], output['fingerprint'] = url, fingerprint

    if input['sample']:
        # e.g. /runs/0/sample.json
        sample_path = os.path.join(index_dirname, input['sample'])
        key_name = '/runs/{run}/{sample}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, sample_path)
        output['sample'] = url

    if input['processed']:
        # e.g. /runs/0/fr/paris.zip
        processed_path = os.path.join(index_dirname, input['processed'])
        package_args = input.get('website') or 'Unknown', input.get('license') or 'Unknown'
        archive_path = util.package_output(source_name, processed_path, *package_args)
        key_name = u'/runs/{run}/{name}.zip'.format(run=run_id, name=source_name)
        url, hash = upload_file(s3, key_name, archive_path)
        output['processed'], output['process hash'] = url, hash

        if os.path.exists(archive_path):
            os.remove(archive_path)

    if input['output']:
        # e.g. /runs/0/output.txt
        output_path = os.path.join(index_dirname, input['output'])
        key_name = '/runs/{run}/{output}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, output_path)
        output['output'] = url

    if input['preview']:
        # e.g. /runs/0/preview.png
        preview_path = os.path.join(index_dirname, input['preview'])
        key_name = '/runs/{run}/{preview}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, preview_path)
        output['preview'] = url

    if input['slippymap']:
        # e.g. /runs/0/slippymap.mbtiles
        slippymap_path = os.path.join(index_dirname, input['slippymap'])
        key_name = '/runs/{run}/{slippymap}'.format(run=run_id, **input)
        url, _ = upload_file(s3, key_name, slippymap_path)
        output['slippymap'] = url

    return output

def do_work(s3, run_id, source_name, job_contents_b64, render_preview, output_dir, mapzen_key=None):
    ''' Do the actual work of running a source file in job_contents.
    '''
    _L.info('Doing work on source {}'.format(repr(source_name)))

    # Make a directory to run the whole job
    workdir = tempfile.mkdtemp(prefix='work-', dir=output_dir)

    # Write the user input to a file
    out_fn = os.path.join(workdir, make_source_filename(source_name))
    with open(out_fn, 'wb') as out_fp:
        out_fp.write(base64.b64decode(job_contents_b64))

    # Make a directory in which to run openaddr
    oa_dir = os.path.join(workdir, 'out')
    os.mkdir(oa_dir)

    # Invoke the job to do
    logfile_path = os.path.join(workdir, 'logfile.txt')
    cmd = 'openaddr-process-one', '-l', logfile_path, out_fn, oa_dir

    if render_preview and mapzen_key:
        cmd += ('--render-preview', '--mapzen-key', mapzen_key)
    else:
        cmd += ('--skip-preview', )

    try:
        known_error, cmd_status = False, 0
        timeout_seconds = JOB_TIMEOUT.seconds + JOB_TIMEOUT.days * 86400
        with open('/dev/null', 'a') as devnull:
            result_stdout = compat.check_output(cmd, timeout=timeout_seconds, stderr=devnull)
    except compat.TimeoutExpired as e:
        known_error, cmd_status, result_stdout = True, None, e.output
    except compat.CalledProcessError as e:
        known_error, cmd_status, result_stdout = True, e.returncode, e.output
    except Exception:
        known_error, cmd_status, result_stdout = False, None, None
        raise
    finally:
        if hasattr(result_stdout, 'decode'):
            # "The actual encoding of the output data may depend on the command
            # being invoked" - https://docs.python.org/3/library/subprocess.html
            result_stdout = result_stdout.decode('utf8', 'replace')

        if known_error:
            # Something went wrong; throw back an error result.
            key_name = '/runs/{run}/logfile.txt'.format(run=run_id)
            try:
                url, _ = upload_file(s3, key_name, logfile_path)
            except IOError:
                output = dict()
            else:
                output = dict(output=url)

            return dict(result_code=cmd_status, result_stdout=result_stdout,
                        message='Something went wrong in {0}'.format(*cmd),
                        output=output)

    result = dict(result_code=0, result_stdout=result_stdout,
                  message=MAGIC_OK_MESSAGE)

    # openaddr-process-one prints a path to index.json
    state_fullpath = result_stdout.strip()

    with open(state_fullpath) as file:
        index = dict(zip(*json.load(file)))

        for key in ('processed', 'sample', 'cache'):
            if not index[key] and not index.get('skipped'):
                result.update(result_code=-1, message='Failed to produce {} data'.format(key))

        index_dirname = os.path.dirname(state_fullpath)
        result['output'] = assemble_output(s3, index, source_name, run_id, index_dirname)

    shutil.rmtree(workdir)
    return result
<|endoftext|>


<commit_before><commit_msg>Changes for ORCIDHUB-476<commit_after><|endoftext|>


<commit_before>import xml.etree.ElementTree as ET
import json
import re
from svtplay_dl.log import log
from svtplay_dl.utils import is_py2, is_py3, decode_html_entities
from svtplay_dl.utils.io import StringIO
from svtplay_dl.output import output
from requests import Session
from requests import __build__ as requests_version

class subtitle(object):
    def __init__(self, options, subtype, url):
        self.url = url
        self.subtitle = None
        self.options = options
        self.subtype = subtype
        self.http = Session()

    def download(self):
        subdata = self.http.request("get", self.url, cookies=self.options.cookies)

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)

        file_d = output(self.options, "srt", mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()

    def tt(self, subdata):
        i = 1
        data = ""
        tree = ET.XML(subdata.text.encode("utf8"))
        xml = tree.find("{http://www.w3.org/2006/10/ttaf1}body").find("{http://www.w3.org/2006/10/ttaf1}div")
        plist = list(xml.findall("{http://www.w3.org/2006/10/ttaf1}p"))
        for node in plist:
            tag = norm(node.tag)
            if tag == "p" or tag == "span":
                begin = node.attrib["begin"]
                if not ("dur" in node.attrib):
                    duration = node.attrib["duration"]
                else:
                    duration = node.attrib["dur"]
                if not ("end" in node.attrib):
                    begin2 = begin.split(":")
                    duration2 = duration.split(":")
                    sec = float(begin2[2]) + float(duration2[2])
                    end = "%02d:%02d:%06.3f" % (int(begin[0]), int(begin[1]), sec)
                else:
                    end = node.attrib["end"]
                data += '%s\n%s --> %s\n' % (i, begin.replace(".", ","), end.replace(".", ","))
                data = tt_text(node, data)
                data += "\n"
                i += 1
        if is_py2:
            data = data.encode("utf8")
        return data

    def json(self, subdata):
        data = json.loads(subdata.text)
        number = 1
        subs = ""
        for i in data:
            subs += "%s\n%s --> %s\n" % (number, timestr(int(i["startMillis"])), timestr(int(i["endMillis"])))
            if is_py2:
                subs += "%s\n\n" % i["text"].encode("utf-8")
            else:
                subs += "%s\n\n" % i["text"]
            number += 1

        return subs

    def sami(self, subdata):
        tree = ET.XML(subdata.text.encode("utf8"))
        subt = tree.find("Font")
        subs = ""
        n = 0
        for i in subt.getiterator():
            if i.tag == "Subtitle":
                n = i.attrib["SpotNumber"]

                if i.attrib["SpotNumber"] == "1":
                    subs += "%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
                else:
                    subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
            else:
                if int(n) > 0:
                    subs += "%s\n" % i.text

        if is_py2:
            subs = subs.encode('utf8')
        return subs

    def smi(self, subdata):
        if requests_version < 0x20300:
            subdata = subdata.content
            if is_py3:
                subdata = subdata.decode("latin")
        else:
            subdata = subdata.text
        ssubdata = StringIO(subdata)
        timea = 0
        number = 1
        data = None
        subs = ""
        TAG_RE = re.compile(r'<[^>]+>')
        bad_char = re.compile(r'\x96')
        for i in ssubdata.readlines():
            i = i.rstrip()
            sync = re.search(r"<SYNC Start=(\d+)>", i)
            if sync:
                if int(sync.group(1)) != int(timea):
                    if data and data != " ":
                        subs += "%s\n%s --> %s\n" % (number, timestr(timea), timestr(sync.group(1)))
                        text = "%s\n" % TAG_RE.sub('', data.replace("
", "\n"))
                        if text[len(text)-2] != "\n":
                            text += "\n"
                        subs += text
                        number += 1
                timea = sync.group(1)
            text = re.search("
(.*)", i)
            if text:
                data = text.group(1)
        recomp = re.compile(r'\r')
        text = bad_char.sub('-', recomp.sub('', subs)).replace('"', '"')
        if is_py2 and isinstance(text, unicode):
            return text.encode("utf-8")
        return text

    def wrst(self, subdata):
        ssubdata = StringIO(subdata.text)
        srt = ""
        subtract = False
        number_b = 1
        number = 0
        block = 0
        subnr = False
        for i in ssubdata.readlines():
            match = re.search(r"^[\r\n]+", i)
            match2 = re.search(r"([\d:\.]+ --> [\d:\.]+)", i)
            match3 = re.search(r"^(\d+)\s", i)
            if i[:6] == "WEBVTT":
                pass
            elif match and number_b > 1:
                block = 0
                srt += "\n"
            elif match2:
                if not subnr:
                    srt += "%s\n" % number_b
                matchx = re.search(r'(\d+):(\d+)[.:]([\d\.]+) --> (\d+):(\d+)[.:]([\d\.]+)', i)
                hour1 = int(matchx.group(1))
                hour2 = int(matchx.group(4))
                if int(number) == 1:
                    if hour1 > 9:
                        subtract = True
                if subtract:
                    hour1 -= 10
                    hour2 -= 10
                time = "%s:%s:%s --> %s:%s:%s\n" % (hour1, matchx.group(2), matchx.group(3).replace(".", ","), hour2, matchx.group(5), matchx.group(6).replace(".", ","))
                srt += time
                block = 1
                subnr = False
                number_b += 1

            elif match3 and block == 0:
                number = match3.group(1)
                srt += "%s\n" % number
                subnr = True
            else:
                sub = re.sub('<[^>]*>', '', i)
                srt += sub.lstrip()
        srt = decode_html_entities(srt)
        if is_py2:
            return srt.encode("utf-8")
        return srt

def timestr(msec):
    """
    Convert a millisecond value to a string of the following
    format:

        HH:MM:SS,SS

    with 10 millisecond precision. Note the , seperator in
    the seconds.
    """
    sec = float(msec) / 1000

    hours = int(sec / 3600)
    sec -= hours * 3600

    minutes = int(sec / 60)
    sec -= minutes * 60

    output = "%02d:%02d:%05.2f" % (hours, minutes, sec)
    return output.replace(".", ",")

def timecolon(data):
    match = re.search(r"(\d+:\d+:\d+):(\d+)", data)
    return "%s,%s" % (match.group(1), match.group(2))

def norm(name):
    if name[0] == "{":
        _, tag = name[1:].split("}")
        return tag
    else:
        return name

def tt_text(node, data):
    if node.text:
        data += "%s\n" % node.text.strip(' \t\n\r')
    for i in node:
        if i.text:
            data += "%s\n" % i.text.strip(' \t\n\r')
        if i.tail:
            text = i.tail.strip(' \t\n\r')
            if text:
                data += "%s\n" % text
    return data
<commit_msg>subtitle_tt: remove namespace info<commit_after>import xml.etree.ElementTree as ET
import json
import re
from svtplay_dl.log import log
from svtplay_dl.utils import is_py2, is_py3, decode_html_entities
from svtplay_dl.utils.io import StringIO
from svtplay_dl.output import output
from requests import Session
from requests import __build__ as requests_version

class subtitle(object):
    def __init__(self, options, subtype, url):
        self.url = url
        self.subtitle = None
        self.options = options
        self.subtype = subtype
        self.http = Session()

    def download(self):
        subdata = self.http.request("get", self.url, cookies=self.options.cookies)

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)

        file_d = output(self.options, "srt", mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()

    def tt(self, subdata):
        i = 1
        data = ""
        if is_py3:
            subs = subdata.text
        else:
            subs = subdata.text.encode("utf8")
        subdata = re.sub(' xmlns="[^"]+"', '', subs, count=1)
        tree = ET.XML(subdata)
        xml = tree.find("body").find("div")
        plist = list(xml.findall("p"))
        for node in plist:
            tag = norm(node.tag)
            if tag == "p" or tag == "span":
                begin = node.attrib["begin"]
                if not ("dur" in node.attrib):
                    duration = node.attrib["duration"]
                else:
                    duration = node.attrib["dur"]
                if not ("end" in node.attrib):
                    begin2 = begin.split(":")
                    duration2 = duration.split(":")
                    sec = float(begin2[2]) + float(duration2[2])
                    end = "%02d:%02d:%06.3f" % (int(begin[0]), int(begin[1]), sec)
                else:
                    end = node.attrib["end"]
                data += '%s\n%s --> %s\n' % (i, begin.replace(".", ","), end.replace(".", ","))
                data = tt_text(node, data)
                data += "\n"
                i += 1
        if is_py2:
            data = data.encode("utf8")
        return data

    def json(self, subdata):
        data = json.loads(subdata.text)
        number = 1
        subs = ""
        for i in data:
            subs += "%s\n%s --> %s\n" % (number, timestr(int(i["startMillis"])), timestr(int(i["endMillis"])))
            if is_py2:
                subs += "%s\n\n" % i["text"].encode("utf-8")
            else:
                subs += "%s\n\n" % i["text"]
            number += 1

        return subs

    def sami(self, subdata):
        tree = ET.XML(subdata.text.encode("utf8"))
        subt = tree.find("Font")
        subs = ""
        n = 0
        for i in subt.getiterator():
            if i.tag == "Subtitle":
                n = i.attrib["SpotNumber"]

                if i.attrib["SpotNumber"] == "1":
                    subs += "%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
                else:
                    subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
            else:
                if int(n) > 0:
                    subs += "%s\n" % i.text

        if is_py2:
            subs = subs.encode('utf8')
        return subs

    def smi(self, subdata):
        if requests_version < 0x20300:
            subdata = subdata.content
            if is_py3:
                subdata = subdata.decode("latin")
        else:
            subdata = subdata.text
        ssubdata = StringIO(subdata)
        timea = 0
        number = 1
        data = None
        subs = ""
        TAG_RE = re.compile(r'<[^>]+>')
        bad_char = re.compile(r'\x96')
        for i in ssubdata.readlines():
            i = i.rstrip()
            sync = re.search(r"<SYNC Start=(\d+)>", i)
            if sync:
                if int(sync.group(1)) != int(timea):
                    if data and data != " ":
                        subs += "%s\n%s --> %s\n" % (number, timestr(timea), timestr(sync.group(1)))
                        text = "%s\n" % TAG_RE.sub('', data.replace("
", "\n"))
                        if text[len(text)-2] != "\n":
                            text += "\n"
                        subs += text
                        number += 1
                timea = sync.group(1)
            text = re.search("
(.*)", i)
            if text:
                data = text.group(1)
        recomp = re.compile(r'\r')
        text = bad_char.sub('-', recomp.sub('', subs)).replace('"', '"')
        if is_py2 and isinstance(text, unicode):
            return text.encode("utf-8")
        return text

    def wrst(self, subdata):
        ssubdata = StringIO(subdata.text)
        srt = ""
        subtract = False
        number_b = 1
        number = 0
        block = 0
        subnr = False
        for i in ssubdata.readlines():
            match = re.search(r"^[\r\n]+", i)
            match2 = re.search(r"([\d:\.]+ --> [\d:\.]+)", i)
            match3 = re.search(r"^(\d+)\s", i)
            if i[:6] == "WEBVTT":
                pass
            elif match and number_b > 1:
                block = 0
                srt += "\n"
            elif match2:
                if not subnr:
                    srt += "%s\n" % number_b
                matchx = re.search(r'(\d+):(\d+)[.:]([\d\.]+) --> (\d+):(\d+)[.:]([\d\.]+)', i)
                hour1 = int(matchx.group(1))
                hour2 = int(matchx.group(4))
                if int(number) == 1:
                    if hour1 > 9:
                        subtract = True
                if subtract:
                    hour1 -= 10
                    hour2 -= 10
                time = "%s:%s:%s --> %s:%s:%s\n" % (hour1, matchx.group(2), matchx.group(3).replace(".", ","), hour2, matchx.group(5), matchx.group(6).replace(".", ","))
                srt += time
                block = 1
                subnr = False
                number_b += 1

            elif match3 and block == 0:
                number = match3.group(1)
                srt += "%s\n" % number
                subnr = True
            else:
                sub = re.sub('<[^>]*>', '', i)
                srt += sub.lstrip()
        srt = decode_html_entities(srt)
        if is_py2:
            return srt.encode("utf-8")
        return srt

def timestr(msec):
    """
    Convert a millisecond value to a string of the following
    format:

        HH:MM:SS,SS

    with 10 millisecond precision. Note the , seperator in
    the seconds.
    """
    sec = float(msec) / 1000

    hours = int(sec / 3600)
    sec -= hours * 3600

    minutes = int(sec / 60)
    sec -= minutes * 60

    output = "%02d:%02d:%05.2f" % (hours, minutes, sec)
    return output.replace(".", ",")

def timecolon(data):
    match = re.search(r"(\d+:\d+:\d+):(\d+)", data)
    return "%s,%s" % (match.group(1), match.group(2))

def norm(name):
    if name[0] == "{":
        _, tag = name[1:].split("}")
        return tag
    else:
        return name

def tt_text(node, data):
    if node.text:
        data += "%s\n" % node.text.strip(' \t\n\r')
    for i in node:
        if i.text:
            data += "%s\n" % i.text.strip(' \t\n\r')
        if i.tail:
            text = i.tail.strip(' \t\n\r')
            if text:
                data += "%s\n" % text
    return data
<|endoftext|>


<commit_before>import xml.etree.ElementTree as ET
import json
import re
from io import StringIO

from svtplay_dl.log import log
from svtplay_dl.utils.text import decode_html_entities
from svtplay_dl.utils.http import HTTP, get_full_url
from svtplay_dl.utils.output import output

from requests import __build__ as requests_version
import platform

class subtitle(object):
    def __init__(self, config, subtype, url, subfix=None, **kwargs):
        self.url = url
        self.subtitle = None
        self.config = config
        self.subtype = subtype
        self.http = HTTP(config)
        self.subfix = subfix
        self.bom = False
        self.output = kwargs.pop("output", None)
        self.kwargs = kwargs

    def __repr__(self):
        return "<Subtitle(type={}, url={}>".format(self.subtype, self.url)

    def download(self):
        subdata = self.http.request("get", self.url)
        if subdata.status_code != 200:
            log.warning("Can't download subtitle file")
            return

        data = None
        if "mtgx" in self.url and subdata.content[:3] == b"\xef\xbb\xbf":
            subdata.encoding = "utf-8"
            self.bom = True

        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            if "tv4play" in self.url and subdata.content[:3] == b"\xef\xbb\xbf":
                subdata.encoding = "utf-8"
                self.bom = True
            if "dplay" in self.url:
                subdata.encoding = "utf-8"
            data = self.wrst(subdata)
        if self.subtype == "wrstsegment":
            data = self.wrstsegment(subdata)
        if self.subtype == "raw":
            data = self.raw(subdata)

        if self.subfix:
            if self.config.get("get_all_subtitles"):
                if self.output["episodename"]:
                    self.output["episodename"] = "{}-{}".format(self.output["episodename"], self.subfix)
                else:
                    self.output["episodename"] = self.subfix

        if self.config.get("get_raw_subtitles"):
            subdata = self.raw(subdata)
            self.save_file(subdata, self.subtype)

        self.save_file(data, "srt")

    def save_file(self, data, subtype):
        if platform.system() == "Windows":
            file_d = output(self.output, self.config, subtype, mode="wt", encoding="utf-8")
        else:
            file_d = output(self.output, self.config, subtype, mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()

    def raw(self, subdata):
        return subdata.text

    def tt(self, subdata):
        i = 1
        data = ""
        subs = subdata.text

        subdata = re.sub(' xmlns="[^"]+"', '', subs, count=1)
        tree = ET.XML(subdata)
        xml = tree.find("body").find("div")
        plist = list(xml.findall("p"))
        for node in plist:
            tag = norm(node.tag)
            if tag == "p" or tag == "span":
                begin = node.attrib["begin"]
                if not ("dur" in node.attrib):
                    duration = node.attrib["duration"]
                else:
                    duration = node.attrib["dur"]
                if not ("end" in node.attrib):
                    begin2 = begin.split(":")
                    duration2 = duration.split(":")
                    try:
                        sec = float(begin2[2]) + float(duration2[2])
                    except ValueError:
                        sec = 0.000
                    end = "%02d:%02d:%06.3f" % (int(begin2[0]), int(begin2[1]), sec)
                else:
                    end = node.attrib["end"]
                data += '%s\n%s --> %s\n' % (i, begin.replace(".", ","), end.replace(".", ","))
                data = tt_text(node, data)
                data += "\n"
                i += 1

        return data

    def json(self, subdata):
        data = json.loads(subdata.text)
        number = 1
        subs = ""
        for i in data:
            subs += "%s\n%s --> %s\n" % (number, timestr(int(i["startMillis"])), timestr(int(i["endMillis"])))
            subs += "%s\n\n" % i["text"]
            number += 1

        return subs

    def sami(self, subdata):
        text = subdata.text
        text = re.sub(r'&', '&', text)
        tree = ET.fromstring(text)
        subt = tree.find("Font")
        subs = ""
        n = 0
        for i in subt.getiterator():
            if i.tag == "Subtitle":
                n = i.attrib["SpotNumber"]

                if i.attrib["SpotNumber"] == "1":
                    subs += "%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
                else:
                    subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
            else:
                if int(n) > 0 and i.text:
                    subs += "%s\n" % decode_html_entities(i.text)

        subs = re.sub('&', r'&', subs)
        return subs

    def smi(self, subdata):
        if requests_version < 0x20300:
            subdata = subdata.content.decode("latin")
        else:
            subdata.encoding = "ISO-8859-1"
            subdata = subdata.text
        ssubdata = StringIO(subdata)
        timea = 0
        number = 1
        data = None
        subs = ""
        TAG_RE = re.compile(r'<(?!\/?i).*?>')
        bad_char = re.compile(r'\x96')
        for i in ssubdata.readlines():
            i = i.rstrip()
            sync = re.search(r"<SYNC Start=(\d+)>", i)
            if sync:
                if int(sync.group(1)) != int(timea):
                    if data and data != " ":
                        subs += "%s\n%s --> %s\n" % (number, timestr(timea), timestr(sync.group(1)))
                        text = "%s\n" % TAG_RE.sub('', data.replace("
", "\n"))
                        text = decode_html_entities(text)
                        if text[len(text) - 2] != "\n":
                            text += "\n"
                        subs += text
                        number += 1
                timea = sync.group(1)
            text = re.search("
(.*)", i)
            if text:
                data = text.group(1)
        recomp = re.compile(r'\r')
        text = bad_char.sub('-', recomp.sub('', subs))
        return text

    def wrst(self, subdata):
        ssubdata = StringIO(subdata.text)
        srt = ""
        subtract = False
        number_b = 1
        number = 0
        block = 0
        subnr = False
        if self.bom:
            ssubdata.read(1)
        for i in ssubdata.readlines():
            match = re.search(r"^[\r\n]+", i)
            match2 = re.search(r"([\d:\.]+ --> [\d:\.]+)", i)
            match3 = re.search(r"^(\d+)\s", i)
            if i[:6] == "WEBVTT":
                continue
            elif "X-TIMESTAMP" in i:
                continue
            elif match and number_b == 1 and self.bom:
                continue
            elif match and number_b > 1:
                block = 0
                srt += "\n"
            elif match2:
                if not subnr:
                    srt += "%s\n" % number_b
                matchx = re.search(r'(?P
\d+):(?P<m1>\d+):(?P<s1>[\d\.]+) --> (?P
\d+):(?P<m2>\d+):(?P<s2>[\d\.]+)', i)
                if matchx:
                    hour1 = int(matchx.group("h1"))
                    hour2 = int(matchx.group("h2"))
                    if int(number) == 1:
                        if hour1 > 9:
                            subtract = True
                    if subtract:
                        hour1 -= 10
                        hour2 -= 10
                else:
                    matchx = re.search(r'(?P<m1>\d+):(?P<s1>[\d\.]+) --> (?P<m2>\d+):(?P<s2>[\d\.]+)', i)
                    hour1 = 0
                    hour2 = 0
                time = "{0:02d}:{1}:{2} --> {3:02d}:{4}:{5}\n".format(hour1, matchx.group("m1"), matchx.group("s1").replace(".", ","),
                                                                      hour2, matchx.group("m2"), matchx.group("s2").replace(".", ","))
                srt += time
                block = 1
                subnr = False
                number_b += 1

            elif match3 and block == 0:
                number = match3.group(1)
                srt += "%s\n" % number
                subnr = True
            else:
                if self.config.get("convert_subtitle_colors"):
                    colors = {'30': '#000000', '31': '#ff0000', '32': '#00ff00', '33': '#ffff00',
                              '34': '#0000ff', '35': '#ff00ff', '36': '#00ffff', '37': '#ffffff'}
                    sub = i
                    for tag, color in colors.items():
                        regex1 = '<' + tag + '>'
                        replace = ''
                        sub = re.sub(regex1, replace, sub)

                    sub = re.sub('</.+>', '', sub)
                else:
                    sub = re.sub('<[^>]*>', '', i)
                srt += sub.strip()
                srt += "\n"
        srt = decode_html_entities(srt)
        return srt

    def wrstsegment(self, subdata):
        time = 0
        subs = []
        for i in self.kwargs["m3u8"].media_segment:
            time += i["EXTINF"]["duration"]
            itemurl = get_full_url(i["URI"], self.url)
            cont = self.http.get(itemurl)
            if "cmore" in self.url:
                cont.encoding = "utf-8"
            text = cont.text.split("\n")
            text = text[3:len(text) - 2]
            if len(text) > 1:
                itmes = []
                for n in text:
                    if n:
                        itmes.append(n)
                    else:
                        if len(subs) > 1 and itmes[1] == subs[-1][1]:  # This will happen when  there is two sections in file
                            ha = strdate(subs[-1][0])
                            ha3 = strdate(itmes[0])
                            second = str2sec(ha3.group(2)) + time
                            subs[-1][0] = "{} --> {}".format(ha.group(1), sec2str(second))
                            itmes = []
                        else:
                            ha = strdate(itmes[0])
                            first = str2sec(ha.group(1)) + time
                            second = str2sec(ha.group(2)) + time
                            itmes[0] = "{} --> {}".format(sec2str(first), sec2str(second))
                            subs.append(itmes)
                            itmes = []
                if itmes:
                    if len(subs) > 0 and itmes[1] == subs[-1][1]:
                        ha = strdate(subs[-1][0])
                        ha3 = strdate(itmes[0])
                        second = str2sec(ha3.group(2)) + time
                        subs[-1][0] = "{} --> {}".format(ha.group(1), sec2str(second))
                    else:
                        ha = strdate(itmes[0])
                        first = str2sec(ha.group(1)) + time
                        second = str2sec(ha.group(2)) + time
                        itmes[0] = "{} --> {}".format(sec2str(first), sec2str(second))
                        subs.append(itmes)

        string = ""
        nr = 1
        for sub in subs:
            string += "{}\n{}\n\n".format(nr, '\n'.join(sub))
            nr += 1

        return string

def timestr(msec):
    """
    Convert a millisecond value to a string of the following
    format:

        HH:MM:SS,SS

    with 10 millisecond precision. Note the , seperator in
    the seconds.
    """
    sec = float(msec) / 1000

    hours = int(sec / 3600)
    sec -= hours * 3600

    minutes = int(sec / 60)
    sec -= minutes * 60

    output = "%02d:%02d:%06.3f" % (hours, minutes, sec)
    return output.replace(".", ",")

def timecolon(data):
    match = re.search(r"(\d+:\d+:\d+):(\d+)", data)
    return "%s,%s" % (match.group(1), match.group(2))

def norm(name):
    if name[0] == "{":
        _, tag = name[1:].split("}")
        return tag
    else:
        return name

def tt_text(node, data):
    if node.text:
        data += "%s\n" % node.text.strip(' \t\n\r')
    for i in node:
        if i.text:
            data += "%s\n" % i.text.strip(' \t\n\r')
        if i.tail:
            text = i.tail.strip(' \t\n\r')
            if text:
                data += "%s\n" % text
    return data

def strdate(datestring):
    match = re.search("^(\d+:\d+:[\.0-9]+) --> (\d+:\d+:[\.0-9]+)", datestring)
    return match

def sec2str(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "{:02d}:{:02d}:{:06.3f}".format(int(h), int(m), s)

def str2sec(string):
    return sum(x * float(t) for x, t in zip([3600, 60, 1], string.split(":")))
<commit_msg>Fix timecodes for segmented subtitles<commit_after>import xml.etree.ElementTree as ET
import json
import re
from io import StringIO

from svtplay_dl.log import log
from svtplay_dl.utils.text import decode_html_entities
from svtplay_dl.utils.http import HTTP, get_full_url
from svtplay_dl.utils.output import output

from requests import __build__ as requests_version
import platform

class subtitle(object):
    def __init__(self, config, subtype, url, subfix=None, **kwargs):
        self.url = url
        self.subtitle = None
        self.config = config
        self.subtype = subtype
        self.http = HTTP(config)
        self.subfix = subfix
        self.bom = False
        self.output = kwargs.pop("output", None)
        self.kwargs = kwargs

    def __repr__(self):
        return "<Subtitle(type={}, url={}>".format(self.subtype, self.url)

    def download(self):
        subdata = self.http.request("get", self.url)
        if subdata.status_code != 200:
            log.warning("Can't download subtitle file")
            return

        data = None
        if "mtgx" in self.url and subdata.content[:3] == b"\xef\xbb\xbf":
            subdata.encoding = "utf-8"
            self.bom = True

        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            if "tv4play" in self.url and subdata.content[:3] == b"\xef\xbb\xbf":
                subdata.encoding = "utf-8"
                self.bom = True
            if "dplay" in self.url:
                subdata.encoding = "utf-8"
            data = self.wrst(subdata)
        if self.subtype == "wrstsegment":
            data = self.wrstsegment(subdata)
        if self.subtype == "raw":
            data = self.raw(subdata)

        if self.subfix:
            if self.config.get("get_all_subtitles"):
                if self.output["episodename"]:
                    self.output["episodename"] = "{}-{}".format(self.output["episodename"], self.subfix)
                else:
                    self.output["episodename"] = self.subfix

        if self.config.get("get_raw_subtitles"):
            subdata = self.raw(subdata)
            self.save_file(subdata, self.subtype)

        self.save_file(data, "srt")

    def save_file(self, data, subtype):
        if platform.system() == "Windows":
            file_d = output(self.output, self.config, subtype, mode="wt", encoding="utf-8")
        else:
            file_d = output(self.output, self.config, subtype, mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()

    def raw(self, subdata):
        return subdata.text

    def tt(self, subdata):
        i = 1
        data = ""
        subs = subdata.text

        subdata = re.sub(' xmlns="[^"]+"', '', subs, count=1)
        tree = ET.XML(subdata)
        xml = tree.find("body").find("div")
        plist = list(xml.findall("p"))
        for node in plist:
            tag = norm(node.tag)
            if tag == "p" or tag == "span":
                begin = node.attrib["begin"]
                if not ("dur" in node.attrib):
                    duration = node.attrib["duration"]
                else:
                    duration = node.attrib["dur"]
                if not ("end" in node.attrib):
                    begin2 = begin.split(":")
                    duration2 = duration.split(":")
                    try:
                        sec = float(begin2[2]) + float(duration2[2])
                    except ValueError:
                        sec = 0.000
                    end = "%02d:%02d:%06.3f" % (int(begin2[0]), int(begin2[1]), sec)
                else:
                    end = node.attrib["end"]
                data += '%s\n%s --> %s\n' % (i, begin.replace(".", ","), end.replace(".", ","))
                data = tt_text(node, data)
                data += "\n"
                i += 1

        return data

    def json(self, subdata):
        data = json.loads(subdata.text)
        number = 1
        subs = ""
        for i in data:
            subs += "%s\n%s --> %s\n" % (number, timestr(int(i["startMillis"])), timestr(int(i["endMillis"])))
            subs += "%s\n\n" % i["text"]
            number += 1

        return subs

    def sami(self, subdata):
        text = subdata.text
        text = re.sub(r'&', '&', text)
        tree = ET.fromstring(text)
        subt = tree.find("Font")
        subs = ""
        n = 0
        for i in subt.getiterator():
            if i.tag == "Subtitle":
                n = i.attrib["SpotNumber"]

                if i.attrib["SpotNumber"] == "1":
                    subs += "%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
                else:
                    subs += "\n%s\n%s --> %s\n" % (i.attrib["SpotNumber"], timecolon(i.attrib["TimeIn"]), timecolon(i.attrib["TimeOut"]))
            else:
                if int(n) > 0 and i.text:
                    subs += "%s\n" % decode_html_entities(i.text)

        subs = re.sub('&', r'&', subs)
        return subs

    def smi(self, subdata):
        if requests_version < 0x20300:
            subdata = subdata.content.decode("latin")
        else:
            subdata.encoding = "ISO-8859-1"
            subdata = subdata.text
        ssubdata = StringIO(subdata)
        timea = 0
        number = 1
        data = None
        subs = ""
        TAG_RE = re.compile(r'<(?!\/?i).*?>')
        bad_char = re.compile(r'\x96')
        for i in ssubdata.readlines():
            i = i.rstrip()
            sync = re.search(r"<SYNC Start=(\d+)>", i)
            if sync:
                if int(sync.group(1)) != int(timea):
                    if data and data != " ":
                        subs += "%s\n%s --> %s\n" % (number, timestr(timea), timestr(sync.group(1)))
                        text = "%s\n" % TAG_RE.sub('', data.replace("
", "\n"))
                        text = decode_html_entities(text)
                        if text[len(text) - 2] != "\n":
                            text += "\n"
                        subs += text
                        number += 1
                timea = sync.group(1)
            text = re.search("
(.*)", i)
            if text:
                data = text.group(1)
        recomp = re.compile(r'\r')
        text = bad_char.sub('-', recomp.sub('', subs))
        return text

    def wrst(self, subdata):
        ssubdata = StringIO(subdata.text)
        srt = ""
        subtract = False
        number_b = 1
        number = 0
        block = 0
        subnr = False
        if self.bom:
            ssubdata.read(1)
        for i in ssubdata.readlines():
            match = re.search(r"^[\r\n]+", i)
            match2 = re.search(r"([\d:\.]+ --> [\d:\.]+)", i)
            match3 = re.search(r"^(\d+)\s", i)
            if i[:6] == "WEBVTT":
                continue
            elif "X-TIMESTAMP" in i:
                continue
            elif match and number_b == 1 and self.bom:
                continue
            elif match and number_b > 1:
                block = 0
                srt += "\n"
            elif match2:
                if not subnr:
                    srt += "%s\n" % number_b
                matchx = re.search(r'(?P
\d+):(?P<m1>\d+):(?P<s1>[\d\.]+) --> (?P
\d+):(?P<m2>\d+):(?P<s2>[\d\.]+)', i)
                if matchx:
                    hour1 = int(matchx.group("h1"))
                    hour2 = int(matchx.group("h2"))
                    if int(number) == 1:
                        if hour1 > 9:
                            subtract = True
                    if subtract:
                        hour1 -= 10
                        hour2 -= 10
                else:
                    matchx = re.search(r'(?P<m1>\d+):(?P<s1>[\d\.]+) --> (?P<m2>\d+):(?P<s2>[\d\.]+)', i)
                    hour1 = 0
                    hour2 = 0
                time = "{0:02d}:{1}:{2} --> {3:02d}:{4}:{5}\n".format(hour1, matchx.group("m1"), matchx.group("s1").replace(".", ","),
                                                                      hour2, matchx.group("m2"), matchx.group("s2").replace(".", ","))
                srt += time
                block = 1
                subnr = False
                number_b += 1

            elif match3 and block == 0:
                number = match3.group(1)
                srt += "%s\n" % number
                subnr = True
            else:
                if self.config.get("convert_subtitle_colors"):
                    colors = {'30': '#000000', '31': '#ff0000', '32': '#00ff00', '33': '#ffff00',
                              '34': '#0000ff', '35': '#ff00ff', '36': '#00ffff', '37': '#ffffff'}
                    sub = i
                    for tag, color in colors.items():
                        regex1 = '<' + tag + '>'
                        replace = ''
                        sub = re.sub(regex1, replace, sub)

                    sub = re.sub('</.+>', '', sub)
                else:
                    sub = re.sub('<[^>]*>', '', i)
                srt += sub.strip()
                srt += "\n"
        srt = decode_html_entities(srt)
        return srt

    def wrstsegment(self, subdata):
        time = 0
        subs = []
        for i in self.kwargs["m3u8"].media_segment:
            itemurl = get_full_url(i["URI"], self.url)
            cont = self.http.get(itemurl)
            if "cmore" in self.url:
                cont.encoding = "utf-8"
            text = cont.text.split("\n")
            for t in text:  # is in text[1] for tv4play, but this should be more future proof
                if 'X-TIMESTAMP-MAP=MPEGTS' in t:
                    time = float(re.search(r"X-TIMESTAMP-MAP=MPEGTS:(\d+)", t).group(1))/90000
            text = text[3:len(text) - 2]
            if len(text) > 1:
                itmes = []
                for n in text:
                    if n:
                        itmes.append(n)
                    else:
                        if len(subs) > 1 and itmes[1] == subs[-1][1]:  # This will happen when  there is two sections in file
                            ha = strdate(subs[-1][0])
                            ha3 = strdate(itmes[0])
                            second = str2sec(ha3.group(2)) + time
                            subs[-1][0] = "{} --> {}".format(ha.group(1), sec2str(second))
                            itmes = []
                        else:
                            ha = strdate(itmes[0])
                            first = str2sec(ha.group(1)) + time
                            second = str2sec(ha.group(2)) + time
                            itmes[0] = "{} --> {}".format(sec2str(first), sec2str(second))
                            subs.append(itmes)
                            itmes = []
                if itmes:
                    if len(subs) > 0 and itmes[1] == subs[-1][1]:
                        ha = strdate(subs[-1][0])
                        ha3 = strdate(itmes[0])
                        second = str2sec(ha3.group(2)) + time
                        subs[-1][0] = "{} --> {}".format(ha.group(1), sec2str(second))
                    else:
                        ha = strdate(itmes[0])
                        first = str2sec(ha.group(1)) + time
                        second = str2sec(ha.group(2)) + time
                        itmes[0] = "{} --> {}".format(sec2str(first), sec2str(second))
                        subs.append(itmes)

        string = ""
        nr = 1
        for sub in subs:
            string += "{}\n{}\n\n".format(nr, '\n'.join(sub))
            nr += 1

        return string

def timestr(msec):
    """
    Convert a millisecond value to a string of the following
    format:

        HH:MM:SS,SS

    with 10 millisecond precision. Note the , seperator in
    the seconds.
    """
    sec = float(msec) / 1000

    hours = int(sec / 3600)
    sec -= hours * 3600

    minutes = int(sec / 60)
    sec -= minutes * 60

    output = "%02d:%02d:%06.3f" % (hours, minutes, sec)
    return output.replace(".", ",")

def timecolon(data):
    match = re.search(r"(\d+:\d+:\d+):(\d+)", data)
    return "%s,%s" % (match.group(1), match.group(2))

def norm(name):
    if name[0] == "{":
        _, tag = name[1:].split("}")
        return tag
    else:
        return name

def tt_text(node, data):
    if node.text:
        data += "%s\n" % node.text.strip(' \t\n\r')
    for i in node:
        if i.text:
            data += "%s\n" % i.text.strip(' \t\n\r')
        if i.tail:
            text = i.tail.strip(' \t\n\r')
            if text:
                data += "%s\n" % text
    return data

def strdate(datestring):
    match = re.search("^(\d+:\d+:[\.0-9]+) --> (\d+:\d+:[\.0-9]+)", datestring)
    return match

def sec2str(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "{:02d}:{:02d}:{:06.3f}".format(int(h), int(m), s)

def str2sec(string):
    return sum(x * float(t) for x, t in zip([3600, 60, 1], string.split(":")))
<|endoftext|>


<commit_before><commit_msg>Update __main__.py<commit_after><|endoftext|>


<commit_before><commit_msg>Added dnf env install, need to go back and fix up this hacky commit later<commit_after><|endoftext|>


<commit_before>import mimetypes
import uuid

from io import BytesIO
from StringIO import StringIO
from os import path

from urlparse import urlparse, urlunparse
from urllib import urlencode

from twisted.internet.protocol import Protocol
from twisted.internet.defer import Deferred
from twisted.python.components import proxyForInterface

from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer, IResponse

from twisted.web.client import (
    Agent,
    FileBodyProducer,
    HTTPConnectionPool,
    RedirectAgent,
    ContentDecoderAgent,
    GzipDecoder
)

from twisted.python.components import registerAdapter

from treq._utils import default_reactor
from treq.auth import add_auth
from treq import multipart

class _BodyBufferingProtocol(Protocol):
    def __init__(self, original, buffer, finished):
        self.original = original
        self.buffer = buffer
        self.finished = finished

    def dataReceived(self, data):
        self.buffer.append(data)
        self.original.dataReceived(data)

    def connectionLost(self, reason):
        self.original.connectionLost(reason)
        self.finished.errback(reason)

class _BufferedResponse(proxyForInterface(IResponse)):
    def __init__(self, original):
        self.original = original
        self._buffer = []
        self._waiters = []
        self._waiting = None
        self._finished = False
        self._reason = None

    def _deliverWaiting(self, reason):
        self._reason = reason
        self._finished = True
        for waiter in self._waiters:
            for segment in self._buffer:
                waiter.dataReceived(segment)
            waiter.connectionLost(reason)

    def deliverBody(self, protocol):
        if self._waiting is None and not self._finished:
            self._waiting = Deferred()
            self._waiting.addBoth(self._deliverWaiting)
            self.original.deliverBody(
                _BodyBufferingProtocol(
                    protocol,
                    self._buffer,
                    self._waiting
                )
            )
        elif self._finished:
            for segment in self._buffer:
                protocol.dataReceived(segment)
            protocol.connectionLost(self._reason)
        else:
            self._waiters.append(protocol)

class HTTPClient(object):
    def __init__(self, agent):
        self._agent = agent

    @classmethod
    def with_config(cls, **kwargs):
        reactor = default_reactor(kwargs.get('reactor'))

        pool = kwargs.get('pool')
        if not pool:
            persistent = kwargs.get('persistent', True)
            pool = HTTPConnectionPool(reactor, persistent=persistent)

        agent = Agent(reactor, pool=pool)

        if kwargs.get('allow_redirects', True):
            agent = RedirectAgent(agent)

        agent = ContentDecoderAgent(agent, [('gzip', GzipDecoder)])

        auth = kwargs.get('auth')
        if auth:
            agent = add_auth(agent, auth)

        return cls(agent)

    def get(self, url, **kwargs):
        return self.request('GET', url, **kwargs)

    def put(self, url, data=None, **kwargs):
        return self.request('PUT', url, data=data, **kwargs)

    def patch(self, url, data=None, **kwargs):
        return self.request('PATCH', url, data=data, **kwargs)

    def post(self, url, data=None, **kwargs):
        return self.request('POST', url, data=data, **kwargs)

    def head(self, url, **kwargs):
        return self.request('HEAD', url, **kwargs)

    def delete(self, url, **kwargs):
        return self.request('DELETE', url, **kwargs)

    def request(self, method, url, **kwargs):
        method = method.upper()

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            url = _combine_query_params(url, params)

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.iteritems():
                    if isinstance(v, str):
                        h.addRawHeader(k, v)
                    else:
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = uuid.uuid4()
            headers.setRawHeaders(
                'content-type', [
                    'multipart/form-data; boundary=%s' % (boundary,)])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(
                data + files, boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(
                    'content-type', ['application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = IBodyProducer(data)

        d = self._agent.request(
            method, url, headers=headers,
            bodyProducer=bodyProducer)

        timeout = kwargs.get('timeout')
        if timeout:
            delayedCall = default_reactor(kwargs.get('reactor')).callLater(
                timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        return d.addCallback(_BufferedResponse)

def _convert_params(params):
    if hasattr(params, "iteritems"):
        return list(sorted(params.iteritems()))
    elif isinstance(params, (tuple, list)):
        return list(params)
    else:
        raise ValueError("Unsupported format")

def _convert_files(files):
    """Files can be passed in a variety of formats:

        * {'file': open("bla.f")}
        * {'file': (name, open("bla.f"))}
        * {'file': (name, content-type, open("bla.f"))}
        * Anything that has iteritems method, e.g. MultiDict:
          MultiDict([(name, open()), (name, open())]

        Our goal is to standardize it to unified form of:

        * [(param, (file name, content type, producer))]
    """

    if hasattr(files, "iteritems"):
        files = files.iteritems()

    for param, val in files:
        file_name, content_type, fobj = (None, None, None)
        if isinstance(val, tuple):
            if len(val) == 2:
                file_name, fobj = val
            elif len(val) == 3:
                file_name, content_type, fobj = val
        else:
            fobj = val
            if hasattr(fobj, "name"):
                file_name = path.basename(fobj.name)

        if not content_type:
            content_type = _guess_content_type(file_name)

        yield (param, (file_name, content_type, IBodyProducer(fobj)))

def _combine_query_params(url, params):
    parsed_url = urlparse(url)

    qs = []

    if parsed_url.query:
        qs.extend([parsed_url.query, '&'])

    qs.append(urlencode(params, doseq=True))

    return urlunparse((parsed_url[0], parsed_url[1],
                       parsed_url[2], parsed_url[3],
                       ''.join(qs), parsed_url[5]))

def _from_bytes(orig_bytes):
    return FileBodyProducer(StringIO(orig_bytes))

def _from_file(orig_file):
    return FileBodyProducer(orig_file)

def _guess_content_type(filename):
    if filename:
        guessed = mimetypes.guess_type(filename)[0]
    else:
        guessed = None
    return guessed or 'application/octet-stream'

registerAdapter(_from_bytes, str, IBodyProducer)
registerAdapter(_from_file, file, IBodyProducer)
registerAdapter(_from_file, StringIO, IBodyProducer)
registerAdapter(_from_file, BytesIO, IBodyProducer)
<commit_msg>Make buffering optional, but default it to on because that is how the content methods used to work.<commit_after>import mimetypes
import uuid

from io import BytesIO
from StringIO import StringIO
from os import path

from urlparse import urlparse, urlunparse
from urllib import urlencode

from twisted.internet.protocol import Protocol
from twisted.internet.defer import Deferred
from twisted.python.components import proxyForInterface

from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer, IResponse

from twisted.web.client import (
    Agent,
    FileBodyProducer,
    HTTPConnectionPool,
    RedirectAgent,
    ContentDecoderAgent,
    GzipDecoder
)

from twisted.python.components import registerAdapter

from treq._utils import default_reactor
from treq.auth import add_auth
from treq import multipart

class _BodyBufferingProtocol(Protocol):
    def __init__(self, original, buffer, finished):
        self.original = original
        self.buffer = buffer
        self.finished = finished

    def dataReceived(self, data):
        self.buffer.append(data)
        self.original.dataReceived(data)

    def connectionLost(self, reason):
        self.original.connectionLost(reason)
        self.finished.errback(reason)

class _BufferedResponse(proxyForInterface(IResponse)):
    def __init__(self, original):
        self.original = original
        self._buffer = []
        self._waiters = []
        self._waiting = None
        self._finished = False
        self._reason = None

    def _deliverWaiting(self, reason):
        self._reason = reason
        self._finished = True
        for waiter in self._waiters:
            for segment in self._buffer:
                waiter.dataReceived(segment)
            waiter.connectionLost(reason)

    def deliverBody(self, protocol):
        if self._waiting is None and not self._finished:
            self._waiting = Deferred()
            self._waiting.addBoth(self._deliverWaiting)
            self.original.deliverBody(
                _BodyBufferingProtocol(
                    protocol,
                    self._buffer,
                    self._waiting
                )
            )
        elif self._finished:
            for segment in self._buffer:
                protocol.dataReceived(segment)
            protocol.connectionLost(self._reason)
        else:
            self._waiters.append(protocol)

class HTTPClient(object):
    def __init__(self, agent):
        self._agent = agent

    @classmethod
    def with_config(cls, **kwargs):
        reactor = default_reactor(kwargs.get('reactor'))

        pool = kwargs.get('pool')
        if not pool:
            persistent = kwargs.get('persistent', True)
            pool = HTTPConnectionPool(reactor, persistent=persistent)

        agent = Agent(reactor, pool=pool)

        if kwargs.get('allow_redirects', True):
            agent = RedirectAgent(agent)

        agent = ContentDecoderAgent(agent, [('gzip', GzipDecoder)])

        auth = kwargs.get('auth')
        if auth:
            agent = add_auth(agent, auth)

        return cls(agent)

    def get(self, url, **kwargs):
        return self.request('GET', url, **kwargs)

    def put(self, url, data=None, **kwargs):
        return self.request('PUT', url, data=data, **kwargs)

    def patch(self, url, data=None, **kwargs):
        return self.request('PATCH', url, data=data, **kwargs)

    def post(self, url, data=None, **kwargs):
        return self.request('POST', url, data=data, **kwargs)

    def head(self, url, **kwargs):
        return self.request('HEAD', url, **kwargs)

    def delete(self, url, **kwargs):
        return self.request('DELETE', url, **kwargs)

    def request(self, method, url, **kwargs):
        method = method.upper()

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            url = _combine_query_params(url, params)

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.iteritems():
                    if isinstance(v, str):
                        h.addRawHeader(k, v)
                    else:
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = uuid.uuid4()
            headers.setRawHeaders(
                'content-type', [
                    'multipart/form-data; boundary=%s' % (boundary,)])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(
                data + files, boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(
                    'content-type', ['application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = IBodyProducer(data)

        d = self._agent.request(
            method, url, headers=headers,
            bodyProducer=bodyProducer)

        timeout = kwargs.get('timeout')
        if timeout:
            delayedCall = default_reactor(kwargs.get('reactor')).callLater(
                timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if kwargs.get('unbuffered', True):
            d.addCallback(_BufferedResponse)

        return d

def _convert_params(params):
    if hasattr(params, "iteritems"):
        return list(sorted(params.iteritems()))
    elif isinstance(params, (tuple, list)):
        return list(params)
    else:
        raise ValueError("Unsupported format")

def _convert_files(files):
    """Files can be passed in a variety of formats:

        * {'file': open("bla.f")}
        * {'file': (name, open("bla.f"))}
        * {'file': (name, content-type, open("bla.f"))}
        * Anything that has iteritems method, e.g. MultiDict:
          MultiDict([(name, open()), (name, open())]

        Our goal is to standardize it to unified form of:

        * [(param, (file name, content type, producer))]
    """

    if hasattr(files, "iteritems"):
        files = files.iteritems()

    for param, val in files:
        file_name, content_type, fobj = (None, None, None)
        if isinstance(val, tuple):
            if len(val) == 2:
                file_name, fobj = val
            elif len(val) == 3:
                file_name, content_type, fobj = val
        else:
            fobj = val
            if hasattr(fobj, "name"):
                file_name = path.basename(fobj.name)

        if not content_type:
            content_type = _guess_content_type(file_name)

        yield (param, (file_name, content_type, IBodyProducer(fobj)))

def _combine_query_params(url, params):
    parsed_url = urlparse(url)

    qs = []

    if parsed_url.query:
        qs.extend([parsed_url.query, '&'])

    qs.append(urlencode(params, doseq=True))

    return urlunparse((parsed_url[0], parsed_url[1],
                       parsed_url[2], parsed_url[3],
                       ''.join(qs), parsed_url[5]))

def _from_bytes(orig_bytes):
    return FileBodyProducer(StringIO(orig_bytes))

def _from_file(orig_file):
    return FileBodyProducer(orig_file)

def _guess_content_type(filename):
    if filename:
        guessed = mimetypes.guess_type(filename)[0]
    else:
        guessed = None
    return guessed or 'application/octet-stream'

registerAdapter(_from_bytes, str, IBodyProducer)
registerAdapter(_from_file, file, IBodyProducer)
registerAdapter(_from_file, StringIO, IBodyProducer)
registerAdapter(_from_file, BytesIO, IBodyProducer)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import config
import sheets_api

import requests
from flask import Flask, request, redirect

import json

app = Flask(__name__)

def send_to_group(message):
    headers = {'content-type': 'application/json; charset=utf-8'}
    to_api = json.dumps({'bot_id': config.BOT_ID, 'text': message})
    resp = requests.post(config.API_URL, to_api, headers=headers)
    print(to_api, repr(resp.text), resp.status_code)

def summary_for_name(name):
    rows = sheets_api.worksheet_keys_to_rows(config.WORKSHEET_KEYS_TO_SCRAPE)
    summary = sheets_api.summary_for_name(rows, name)
    return summary or 'Sorry, no match found for "%s".' % name

@app.route('/', methods=['POST'])
def posted():
    data = request.get_json()
    text = data['text']
    clean = text.lower().strip()
    if clean.startswith(config.COMMAND_PREFIX):
        split_point = len(config.COMMAND_PREFIX)
        query = text[split_point:].strip()
        print('Searching:', repr(query))
        message = 'Searching for "%s"...' % query
        send_to_group(message)
        print('Sent message')
        print(repr(summary_for_name(query)))
        send_to_group(summary_for_name(query))
        print('Sent final')
        return ''
    else:
        print('Ignored:  ', repr(text))
        return ''

@app.route('/', methods=['GET'])
def about_me():
    return redirect(config.INDEX_REDIRECT_URL)

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Remove debug lines<commit_after>#!/usr/bin/env python3

import config
import sheets_api

import requests
from flask import Flask, request, redirect

import json

app = Flask(__name__)

def send_to_group(message):
    headers = {'content-type': 'application/json; charset=utf-8'}
    to_api = json.dumps({'bot_id': config.BOT_ID, 'text': message})
    resp = requests.post(config.API_URL, to_api, headers=headers)
    print(to_api, repr(resp.text), resp.status_code)

def summary_for_name(name):
    rows = sheets_api.worksheet_keys_to_rows(config.WORKSHEET_KEYS_TO_SCRAPE)
    summary = sheets_api.summary_for_name(rows, name)
    return summary or 'Sorry, no match found for "%s".' % name

@app.route('/', methods=['POST'])
def posted():
    data = request.get_json()
    text = data['text']
    clean = text.lower().strip()
    if clean.startswith(config.COMMAND_PREFIX):
        split_point = len(config.COMMAND_PREFIX)
        query = text[split_point:].strip()
        print('Searching:', repr(query))
        message = 'Searching for "%s"...' % query
        send_to_group(message)
        send_to_group(summary_for_name(query))
        return ''
    else:
        print('Ignored:  ', repr(text))
        return ''

@app.route('/', methods=['GET'])
def about_me():
    return redirect(config.INDEX_REDIRECT_URL)

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>"""
<Program Name>
  update_and_build.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud

<Purpose>
  Builds an installer from the latest code and in the process
  copies the proper files to the directory that the software
  updaters check for updates.
"""

import make_base_installers
import os
import sys
import shutil

# For unique temp folders
PROGRAM_NAME = "build_and_update"
UPDATER_SITE = "/home/couvb/public_html/updatesite"
#UPDATER_SITE = "/homes/iws/butaud/public_html/updatesite"
INSTALL_DIR = "seattle_repy"
DIST_DIR = "/var/www/dist"
#DIST_DIR = "/homes/iws/butaud/www/dist"

DEBUG = True

def main():
    if len(sys.argv) < 2:
        print "usage: python build_and_update.py trunk/location/ [version]"
    else:
        # Confirm that the user really wants to update all the software
        print "This will update all of the client software!"
        print "Are you sure you want to proceed? (y/n)"
        if raw_input()[0].lower() == "y":
            if len(sys.argv) < 3:
                build_and_update(sys.argv[1])
            else:
                build_and_update(sys.argv[1], sys.argv[2])
        else:
            print "Use make_base_installers instead to just create installers."

def build_and_update(trunk_location, version=""):
    """
    <Purpose>
      Builds the installers, copying the proper files to the software updater
      directory in the process, and deposits the installers in the dist
      directory. Only meant to be run on the actual seattle server.

    <Arguments>
      trunk_location:
        The path to the repository's trunk directory.
      version:
        The version of the distribution, which will be appended to the
        base name of each installer.

    <Exceptions>
      IOError on bad filepaths.

    <Side Effects>
      None.

    <Returns>
      None.
    """
    if not os.path.exists(trunk_location):
        raise IOError("Trunk could not be found at " + trunk_location)
    update_dir = UPDATER_SITE + "/real"
    if DEBUG:
        update_dir = UPDATER_SITE + "/test"
    # First, make the temp directories
    temp_dir = "/tmp/" + PROGRAM_NAME + str(os.getpid())
    if not os.path.exists(temp_dir):
        os.mkdir(temp_dir)
    install_dir = temp_dir + "/" + INSTALL_DIR
    if os.path.exists(install_dir):
        shutil.rmtree(install_dir)
    os.mkdir(install_dir)
    # Next, prepare the installation files
    make_base_installers.prepare_initial_files(trunk_location, install_dir)

    # Copy the files to the updater site
    os.popen("rm -f " + update_dir + "/*")
    os.popen("cp " + install_dir + "/* " + update_dir)

    # Finish preparing the installation files
    make_base_installers.prepare_final_files(trunk_location, install_dir)

    # Now, package each installer
    dist_dir = trunk_location + "/dist"
    make_base_installers.package_win(dist_dir, install_dir, make_base_installers.get_inst_name("win", version), DIST_DIR)
    make_base_installers.package_linux(dist_dir, install_dir, make_base_installers.get_inst_name("linux", version), DIST_DIR)
    make_base_installers.package_mac(dist_dir, install_dir, make_base_installers.get_inst_name("mac", version), DIST_DIR)

    # Remove the temp directory when done.
    shutil.rmtree(temp_dir)
    print "Done!"

if __name__ == "__main__":
    main()
<commit_msg>Took update_and_build.py out of debug mode<commit_after>"""
<Program Name>
  update_and_build.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud

<Purpose>
  Builds an installer from the latest code and in the process
  copies the proper files to the directory that the software
  updaters check for updates.
"""

import make_base_installers
import os
import sys
import shutil

# For unique temp folders
PROGRAM_NAME = "build_and_update"
UPDATER_SITE = "/home/couvb/public_html/updatesite"
#UPDATER_SITE = "/homes/iws/butaud/public_html/updatesite"
INSTALL_DIR = "seattle_repy"
DIST_DIR = "/var/www/dist"
#DIST_DIR = "/homes/iws/butaud/www/dist"

DEBUG = False

def main():
    if len(sys.argv) < 2:
        print "usage: python build_and_update.py trunk/location/ [version]"
    else:
        # Confirm that the user really wants to update all the software
        print "This will update all of the client software!"
        print "Are you sure you want to proceed? (y/n)"
        if raw_input()[0].lower() == "y":
            if len(sys.argv) < 3:
                build_and_update(sys.argv[1])
            else:
                build_and_update(sys.argv[1], sys.argv[2])
        else:
            print "Use make_base_installers instead to just create installers."

def build_and_update(trunk_location, version=""):
    """
    <Purpose>
      Builds the installers, copying the proper files to the software updater
      directory in the process, and deposits the installers in the dist
      directory. Only meant to be run on the actual seattle server.

    <Arguments>
      trunk_location:
        The path to the repository's trunk directory.
      version:
        The version of the distribution, which will be appended to the
        base name of each installer.

    <Exceptions>
      IOError on bad filepaths.

    <Side Effects>
      None.

    <Returns>
      None.
    """
    if not os.path.exists(trunk_location):
        raise IOError("Trunk could not be found at " + trunk_location)
    update_dir = UPDATER_SITE + "/real"
    if DEBUG:
        update_dir = UPDATER_SITE + "/test"
    # First, make the temp directories
    temp_dir = "/tmp/" + PROGRAM_NAME + str(os.getpid())
    if not os.path.exists(temp_dir):
        os.mkdir(temp_dir)
    install_dir = temp_dir + "/" + INSTALL_DIR
    if os.path.exists(install_dir):
        shutil.rmtree(install_dir)
    os.mkdir(install_dir)
    # Next, prepare the installation files
    make_base_installers.prepare_initial_files(trunk_location, install_dir)

    # Copy the files to the updater site
    os.popen("rm -f " + update_dir + "/*")
    os.popen("cp " + install_dir + "/* " + update_dir)

    # Finish preparing the installation files
    make_base_installers.prepare_final_files(trunk_location, install_dir)

    # Now, package each installer
    dist_dir = trunk_location + "/dist"
    make_base_installers.package_win(dist_dir, install_dir, make_base_installers.get_inst_name("win", version), DIST_DIR)
    make_base_installers.package_linux(dist_dir, install_dir, make_base_installers.get_inst_name("linux", version), DIST_DIR)
    make_base_installers.package_mac(dist_dir, install_dir, make_base_installers.get_inst_name("mac", version), DIST_DIR)

    # Remove the temp directory when done.
    shutil.rmtree(temp_dir)
    print "Done!"

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
<Program Name>
  update_and_build.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud
  Zachary Boka

<Purpose>
  Builds an installer from the latest code and in the process
  copies the proper files to the directory that the software
  updaters check for updates.
"""

import make_base_installers
import os
import sys
import shutil
import subprocess
import tempfile

DEBUG = False

# For unique temp folders
PROGRAM_NAME = "build_and_update"
UPDATER_SITE = "/home/couvb/public_html/updatesite/test"
if DEBUG:
    UPDATER_SITE = "/home/butaud/temp/updatesite/test"
INSTALL_DIR = "seattle_repy"
DIST_DIR = "/var/www/dist"
if DEBUG:
    DIST_DIR = "/home/butaud/temp/dist"

def get_digits(s):
    """
    Utility function, returns all the digits from the given string.
    """
    retval = ""
    for c in s:
        if c.isdigit():
            retval += c
    return retval

def sanity_checks(trunk_location, updater_site, pubkey, version):
    """
    <Purpose>
      Checks to make sure that the keys and updater site in the softwareupdater
      script are the same as the ones that this script was given, and that the
      version's release is the same as the version specified in the nodemanager
      script.

    <Arguments>
      trunk_location:
        The path to the repository's trunk, used to find the softwareupdater
        and nodemanager scripts.
      updater_site:
        The local directory where the update is going to be pushed to.
      pubkey:
        Path to the public key that will be used when building the metainfo file
        for the installers and the softwareupdater.
      version:
        Version of the intended release.

    <Exceptions>
      IOError if the softwareupdater script, nodemanager script, or public key
      are not found.

    <Side Effects>
      None.

    <Returns>
      A tuple representing which checks passed and failed (True for pass,
      for for fail). The first element represents the url check, the second
      represents the public key check, and the third represents the version
      check.
    """
    updater_path = os.path.realpath(trunk_location +
                                    "/softwareupdater/softwareupdater.mix")
    nm_path = os.path.realpath(trunk_location + "/nodemanager/nmmain.py")

    if not os.path.exists(updater_path):
        raise IOError("Could not find softwareupdater script at " +
                      updater_path)
    if not os.path.exists(nm_path):
        raise IOError("Could not find nodemanager script at " + nm_path)

    if not os.path.exists(pubkey):
        raise IOError("Could not find public key at " +
                      os.path.realpath(pubkey))

    updater_site = os.path.realpath(updater_site)

    # Find the values in the softwareupdater script
    script_f = open(updater_path)
    script_url = None
    script_key = []
    for line in script_f:
        if "softwareurl = " in line:
            script_url = line.split('"')[1]
        if "softwareupdatepublickey = " in line:
            script_key.append(get_digits(line.split(",")[0]))
            script_key.append(get_digits(line.split(",")[1]))
    script_f.close()

    # Find the version in the nodemanager script
    script_f = open(nm_path)
    script_version = None
    for line in script_f:
        if "version = " in line:
            script_version = line.split('"')[1]
    script_f.close()

    # Get the public key info from the given key.
    key_f = open(pubkey)
    given_key = []
    for line in key_f:
        if line:
            given_key.append(line.split(" ")[0])
            given_key.append(line.split(" ")[1])

    # Get the relevant part of the script url (everything that comes
    # after "couvb")
    script_url_end = script_url[script_url.find("couvb") + len("couvb"):]
    if script_url_end[-1] == "/":
        script_url_end = script_url_end[:-1]

    # Check and see if the updater path ends the same way.
    urls_match = True
    if not updater_site.endswith(script_url_end):
        urls_match = False

    # Check to see if the script key and the given key match,
    keys_match = True
    if not (script_key[0] == given_key[0] and script_key[1] == given_key[1]):
        keys_match = False

    # Check to see if the script version and the given version match.
    versions_match = True
    if not script_version == version:
        versions_match = False

    return (urls_match, keys_match, versions_match)

def build_and_update(trunk_location, pubkey, privkey, version):
    """
    <Purpose>
      Builds the installers, copying the proper files to the software updater
      directory in the process, and deposits the installers in the dist
      directory. Only meant to be run on the actual seattle server.

    <Arguments>
      trunk_location:
        The path to the repository's trunk directory.
      pubkey:
        The public key to be used in the installers.
      privkey:
        The private key to be used in the installers.
      version:
        The version of the distribution, which will be appended to the
        base name of each installer.

    <Exceptions>
      IOError on bad filepaths.

    <Side Effects>
      None.

    <Returns>
      None.
    """
    if not os.path.exists(trunk_location):
        raise IOError("Trunk could not be found at " + trunk_location)
    update_dir = UPDATER_SITE

    # Perform some sanity checks on to make sure the softwareupdater script
    # matches up with the specified info.
    urls_match, keys_match, versions_match = sanity_checks(trunk_location,
                                                           update_dir, pubkey,
                                                           version)
    if not (urls_match and keys_match and versions_match):
        if not urls_match:
            print "Updater location does not match with url in softwareupdater.mix."
        if not keys_match:
            print "Given public key does not match with key in softwareupdater.mix."
        if not versions_match:
            print "Given version does not match with version in nmmain.py"
        return
    print "Sanity checks passed."

    # First, make the temp directories

    # Zack: Create temporary directory for the installation
    temp_install_dir = tempfile.mkdtemp()
    # Zack: Create temporary directory for creating the tarball(s)
    temp_tarball_dir = tempfile.mkdtemp()

    # Next, prepare the installation files
    install_files = make_base_installers.prepare_gen_install_files(trunk_location,
                                                   temp_install_dir, False,
                                                   pubkey, privkey, True)

    # Copy the essential files to the updater site
    p = subprocess.Popen("rm -f " + update_dir + "/*", shell=True)
    p.wait()
    # Zack: Exclude the files that did not belong in the metafile
    for fname in install_files:
      if "nodeman.cfg" != fname and "resources.offcut" != fname
         and "install.py" != fname:
         shutil.copy2(fname, update_dir)

    # Now, package each installer
    make_base_installers.package_win_or_winmob(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("win", version), install_files)
    make_base_installers.package_linux_or_mac(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("linux", version), install_files)
    make_base_installers.package_linux_or_mac(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("mac", version), install_files)

    # Zack: Move the installers to the specified output directory
    os.chdir(temp_tarball_dir)
    for tarball in os.listdir(temp_tarball_dir):
      shutil.copy2(tarball, DIST_DIR)

    # Copy each versioned installer name over the corresponding base
    # base installer
    for dist in ["win", "linux", "mac"]:
        versioned_name = make_base_installers.get_inst_name(dist, version)
        unversioned_name = make_base_installers.get_inst_name(dist, "")
        shutil.copy2(DIST_DIR + "/" + versioned_name, DIST_DIR + "/" + unversioned_name)

    # Zack: Remove the temp directory when done.
    shutil.rmtree(temp_install_dir)
    shutil.rmtree(temp_tarball_dir)
    print "Done!"

def main():
    if len(sys.argv) < 5:
        print "usage: python update_and_build.py trunk/location/ publickey privatekey version"
    else:
        if not DEBUG:
            # Confirm that the user really wants to update all the software
            print "This will update all of the client software!"
            print "Are you sure you want to proceed? (y/n)"
            if raw_input()[0].lower() == "y":
                build_and_update(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
            else:
                print "Use make_base_installers instead to just create installers."
        else:
            # If we're in DEBUG mode, don't bother to confirm
            print "Operating in debug mode..."
            build_and_update(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])

if __name__ == "__main__":
    main()
<commit_msg>Committing revised update_and_build.py -- this program works now, but may still need to be cleaned up a bit<commit_after>"""
<Program Name>
  update_and_build.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud
  Zachary Boka

<Purpose>
  Builds an installer from the latest code and in the process
  copies the proper files to the directory that the software
  updaters check for updates.
"""

import make_base_installers
import os
import sys
import shutil
import subprocess
import tempfile

DEBUG = False

# For unique temp folders
PROGRAM_NAME = "build_and_update"
UPDATER_SITE = "/home/couvb/public_html/updatesite/test"
if DEBUG:
    UPDATER_SITE = "/home/zackrb/test/updatesite/"
INSTALL_DIR = "seattle_repy"
DIST_DIR = "/var/www/dist"
if DEBUG:
    DIST_DIR = "/home/test/dist/"

def get_digits(s):
    """
    Utility function, returns all the digits from the given string.
    """
    retval = ""
    for c in s:
        if c.isdigit():
            retval += c
    return retval

def sanity_checks(trunk_location, updater_site, pubkey, version):
    """
    <Purpose>
      Checks to make sure that the keys and updater site in the softwareupdater
      script are the same as the ones that this script was given, and that the
      version's release is the same as the version specified in the nodemanager
      script.

    <Arguments>
      trunk_location:
        The path to the repository's trunk, used to find the softwareupdater
        and nodemanager scripts.
      updater_site:
        The local directory where the update is going to be pushed to.
      pubkey:
        Path to the public key that will be used when building the metainfo file
        for the installers and the softwareupdater.
      version:
        Version of the intended release.

    <Exceptions>
      IOError if the softwareupdater script, nodemanager script, or public key
      are not found.

    <Side Effects>
      None.

    <Returns>
      A tuple representing which checks passed and failed (True for pass,
      for for fail). The first element represents the url check, the second
      represents the public key check, and the third represents the version
      check.
    """
    updater_path = os.path.realpath(trunk_location +
                                    "/softwareupdater/softwareupdater.mix")
    nm_path = os.path.realpath(trunk_location + "/nodemanager/nmmain.py")

    if not os.path.exists(updater_path):
        raise IOError("Could not find softwareupdater script at " +
                      updater_path)
    if not os.path.exists(nm_path):
        raise IOError("Could not find nodemanager script at " + nm_path)

    if not os.path.exists(pubkey):
        raise IOError("Could not find public key at " +
                      os.path.realpath(pubkey))

    updater_site = os.path.realpath(updater_site)

    # Find the values in the softwareupdater script
    script_f = open(updater_path)
    script_url = None
    script_key = []
    for line in script_f:
        if "softwareurl = " in line:
            script_url = line.split('"')[1]
        if "softwareupdatepublickey = " in line:
            script_key.append(get_digits(line.split(",")[0]))
            script_key.append(get_digits(line.split(",")[1]))
    script_f.close()

    # Find the version in the nodemanager script
    script_f = open(nm_path)
    script_version = None
    for line in script_f:
        if "version = " in line:
            script_version = line.split('"')[1]
    script_f.close()

    # Get the public key info from the given key.
    key_f = open(pubkey)
    given_key = []
    for line in key_f:
        if line:
            given_key.append(line.split(" ")[0])
            given_key.append(line.split(" ")[1])

    # Get the relevant part of the script url (everything that comes
    # after "couvb")
    script_url_end = script_url[script_url.find("couvb") + len("couvb"):]
    if script_url_end[-1] == "/":
        script_url_end = script_url_end[:-1]

    # Check and see if the updater path ends the same way.
    urls_match = True
    if not updater_site.endswith(script_url_end):
        urls_match = False

    # Check to see if the script key and the given key match,
    keys_match = True
    if not (script_key[0] == given_key[0] and script_key[1] == given_key[1]):
        keys_match = False

    # Check to see if the script version and the given version match.
    versions_match = True
    if not script_version == version:
        versions_match = False

    return (urls_match, keys_match, versions_match)

def build_and_update(trunk_location, pubkey, privkey, version):
    """
    <Purpose>
      Builds the installers, copying the proper files to the software updater
      directory in the process, and deposits the installers in the dist
      directory. Only meant to be run on the actual seattle server.

    <Arguments>
      trunk_location:
        The path to the repository's trunk directory.
      pubkey:
        The public key to be used in the installers.
      privkey:
        The private key to be used in the installers.
      version:
        The version of the distribution, which will be appended to the
        base name of each installer.

    <Exceptions>
      IOError on bad filepaths.

    <Side Effects>
      None.

    <Returns>
      None.
    """
    if not os.path.exists(trunk_location):
        raise IOError("Trunk could not be found at " + trunk_location)
    update_dir = UPDATER_SITE

    # Perform some sanity checks on to make sure the softwareupdater script
    # matches up with the specified info.
    urls_match, keys_match, versions_match = sanity_checks(trunk_location,
                                                           update_dir, pubkey,
                                                           version)
    if not (urls_match and keys_match and versions_match):
        if not urls_match:
            print "Updater location does not match with url in softwareupdater.mix."
        if not keys_match:
            print "Given public key does not match with key in softwareupdater.mix."
        if not versions_match:
            print "Given version does not match with version in nmmain.py"
        return
    print "Sanity checks passed."

    # First, make the temp directories

    # Zack: Create temporary directory for the installation
    temp_install_dir = tempfile.mkdtemp()
    # Zack: Create temporary directory for creating the tarball(s)
    temp_tarball_dir = tempfile.mkdtemp()

    # Next, prepare the installation files
    install_files = make_base_installers.prepare_gen_install_files(trunk_location,
                                                   temp_install_dir, False,
                                                   pubkey, privkey, True)

    # Copy the essential files to the updater site
    p = subprocess.Popen("rm -f " + update_dir + "/*", shell=True)
    p.wait()
    # Zack: Exclude the files that did not belong in the metafile
    for fname in install_files:
      if "nodeman.cfg" != fname and "resources.offcut" != fname
         and "install.py" != fname:
         shutil.copy2(fname, update_dir)

    # Now, package each installer
    make_base_installers.package_win_or_winmob(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("win", version), install_files)
    make_base_installers.package_linux_or_mac(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("linux", version), install_files)
    make_base_installers.package_linux_or_mac(trunk_location, temp_install_dir, temp_tarball_dir, make_base_installers.get_inst_name("mac", version), install_files)

    # Zack: Move the installers to the specified output directory
    os.chdir(temp_tarball_dir)
    for tarball in os.listdir(temp_tarball_dir):
      shutil.copy2(tarball, DIST_DIR)

    # Copy each versioned installer name over the corresponding base
    # base installer
    for dist in ["win", "linux", "mac"]:
        versioned_name = make_base_installers.get_inst_name(dist, version)
        unversioned_name = make_base_installers.get_inst_name(dist, "")
        shutil.copy2(DIST_DIR + "/" + versioned_name, DIST_DIR + "/" + unversioned_name)

    # Zack: Remove the temp directory when done.
    shutil.rmtree(temp_install_dir)
    shutil.rmtree(temp_tarball_dir)
    print "Done!"

def main():
    if len(sys.argv) < 5:
        print "usage: python update_and_build.py trunk/location/ publickey privatekey version"
    else:
        trunk_location = os.path.realpath(sys.argv[1])
        pubkey = os.path.realpath(sys.argv[2])
        privkey = os.path.realpath(sys.argv[3])
        if not DEBUG:
            # Confirm that the user really wants to update all the software
            print "This will update all of the client software!"
            print "Are you sure you want to proceed? (y/n)"
            if raw_input()[0].lower() == "y":
                build_and_update(trunk_location, pubkey, privkey, sys.argv[4])
            else:
                print "Use make_base_installers instead to just create installers."
        else:
            # If we're in DEBUG mode, don't bother to confirm
            print "Operating in debug mode..."
            build_and_update(trunk_location, pubkey, privkey, sys.argv[4])

if __name__ == "__main__":
    main()
<|endoftext|>