<commit_before>#!/usr/bin/env python3
"""A simple command line application to download youtube videos."""

import argparse
import datetime as dt
import gzip
import json
import logging
import os
import shutil
import sys
import subprocess # nosec
from io import BufferedWriter
from typing import Any, Optional, List
from pathlib import Path

from pytube import __version__, CaptionQuery, Stream, Playlist
from pytube import YouTube
from pytube.exceptions import PytubeError
from pytube.helpers import safe_filename

logger = logging.getLogger(__name__)

def main():
    """Command line application to download youtube videos."""
    # noinspection PyTypeChecker
    parser = argparse.ArgumentParser(description=main.__doc__)
    args = _parse_args(parser)
    logging.getLogger().setLevel(max(3 - args.verbosity, 0) * 10)

    if not args.url or "youtu" not in args.url:
        parser.print_help()
        sys.exit(1)

    if "/playlist" in args.url:
        print("Loading playlist...")
        playlist = Playlist(args.url)
        if not args.target:
            args.target = safe_filename(playlist.title())
        for youtube_video in playlist.videos:
            try:
                _perform_args_on_youtube(youtube_video, args)
            except PytubeError as e:
                print(f"There was an error with video: {youtube_video}")
                print(e)
    else:
        print("Loading video...")
        youtube = YouTube(args.url)
        _perform_args_on_youtube(youtube, args)

def _perform_args_on_youtube(youtube: YouTube, args: argparse.Namespace) -> None:
    if args.list:
        display_streams(youtube)
    if args.build_playback_report:
        build_playback_report(youtube)
    if args.itag:
        download_by_itag(youtube=youtube, itag=args.itag, target=args.target)
    if hasattr(args, "caption_code"):
        download_caption(
            youtube=youtube, lang_code=args.caption_code, target=args.target
        )
    if args.resolution:
        download_by_resolution(
            youtube=youtube, resolution=args.resolution, target=args.target
        )
    if args.audio:
        download_audio(youtube=youtube, filetype=args.audio, target=args.target)
    if args.ffmpeg:
        ffmpeg_process(youtube=youtube, resolution=args.ffmpeg, target=args.target)

def _parse_args(
    parser: argparse.ArgumentParser, args: Optional[List] = None
) -> argparse.Namespace:
    parser.add_argument("url", help="The YouTube /watch or /playlist url", nargs="?")
    parser.add_argument(
        "--version", action="version", version="%(prog)s " + __version__,
    )
    parser.add_argument(
        "--itag", type=int, help="The itag for the desired stream",
    )
    parser.add_argument(
        "-r", "--resolution", type=str, help="The resolution for the desired stream",
    )
    parser.add_argument(
        "-l",
        "--list",
        action="store_true",
        help=(
            "The list option causes pytube cli to return a list of streams "
            "available to download"
        ),
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        dest="verbosity",
        help="Verbosity level",
    )
    parser.add_argument(
        "--build-playback-report",
        action="store_true",
        help="Save the html and js to disk",
    )
    parser.add_argument(
        "-c",
        "--caption-code",
        type=str,
        default=argparse.SUPPRESS,
        nargs="?",
        help=(
            "Download srt captions for given language code. "
            "Prints available language codes if no argument given"
        ),
    )
    parser.add_argument(
        "-t",
        "--target",
        help=(
            "The output directory for the downloaded stream. "
            "Default is current working directory"
        ),
    )
    parser.add_argument(
        "-a",
        "--audio",
        const="mp4",
        nargs="?",
        help=(
            "Download the audio for a given URL at the highest bitrate available"
            "Defaults to mp4 format if none is specified"
        ),
    )
    parser.add_argument(
        "-f",
        "--ffmpeg",
        const="best",
        nargs="?",
        help=(
            "If no resolution provided, will download the highest quality video and\
            audio streams avaiable, will try to download mp4 but will then download\
            a webm if none are found"
        ),
    )

    return parser.parse_args(args)

def build_playback_report(youtube: YouTube) -> None:
    """Serialize the request data to json for offline debugging.

    :param YouTube youtube:
        A YouTube object.
    """
    ts = int(dt.datetime.utcnow().timestamp())
    fp = os.path.join(os.getcwd(), f"yt-video-{youtube.video_id}-{ts}.json.gz")

    js = youtube.js
    watch_html = youtube.watch_html
    vid_info = youtube.vid_info

    with gzip.open(fp, "wb") as fh:
        fh.write(
            json.dumps(
                {
                    "url": youtube.watch_url,
                    "js": js,
                    "watch_html": watch_html,
                    "video_info": vid_info,
                }
            ).encode("utf8"),
        )

def display_progress_bar(
    bytes_received: int, filesize: int, ch: str = "█", scale: float = 0.55
) -> None:
    """Display a simple, pretty progress bar.

    Example:
    ~~~~~~~~
    PSY - GANGNAM STYLE(강남스타일) MV.mp4
    ↳ |███████████████████████████████████████| 100.0%

    :param int bytes_received:
        The delta between the total file size (bytes) and bytes already
        written to disk.
    :param int filesize:
        File size of the media stream in bytes.
    :param str ch:
        Character to use for presenting progress segment.
    :param float scale:
        Scale multiplier to reduce progress bar size.

    """
    columns = shutil.get_terminal_size().columns
    max_width = int(columns * scale)

    filled = int(round(max_width * bytes_received / float(filesize)))
    remaining = max_width - filled
    progress_bar = ch * filled + " " * remaining
    percent = round(100.0 * bytes_received / float(filesize), 1)
    text = f" ↳ |{progress_bar}| {percent}%\r"
    sys.stdout.write(text)
    sys.stdout.flush()

# noinspection PyUnusedLocal
def on_progress(
    stream: Any, chunk: Any, file_handler: BufferedWriter, bytes_remaining: int
) -> None:
    filesize = stream.filesize
    bytes_received = filesize - bytes_remaining
    display_progress_bar(bytes_received, filesize)

def _download(
    stream: Stream, target: Optional[str] = None, filename: Optional[str] = None
) -> None:
    filesize_megabytes = stream.filesize // 1048576
    print(f"{stream.default_filename} | {filesize_megabytes} MB")
    stream.download(output_path=target, filename=filename)
    sys.stdout.write("\n")

def unique_name(base: str, subtype: str, video_audio: str, target: Path) -> str:
    """
    Given a base name, the file format, and the target directory, will generate
    a filename unique for that directory and file format.
    :param str base:
        The given base-name.
    :param str subtype:
        The filetype of the video which will be downloaded.
    :param Path target:
        Target directory for download.
    """
    counter = 0
    while True:
        name = f"{base}_{video_audio}_{counter}"
        unique = target / f"{name}.{subtype}"

        if not unique.exists():
            return str(name)
        counter += 1

def ffmpeg_process(
    youtube: YouTube, resolution: str, target: Optional[str] = None
) -> None:
    """
    Decides the correct video stream to download, then calls ffmpeg_downloader.

    :param YouTube youtube:
        A valid YouTube object.
    :param str resolution:
        YouTube video resolution.
    :param str target:
        Target directory for download
    """
    youtube.register_on_progress_callback(on_progress)
    if target is None:
        target = Path.cwd()
    else:
        target = Path(target)

    if resolution == "best":
        highest_quality = (
            youtube.streams.filter(progressive=False)
            .order_by("resolution")
            .desc()
            .first()
        )

        video_stream = (
            youtube.streams.filter(progressive=False, subtype="mp4")
            .order_by("resolution")
            .desc()
            .first()
        )

        if highest_quality.resolution == video_stream.resolution:
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)
        else:
            ffmpeg_downloader(youtube=youtube, stream=highest_quality, target=target)
    else:
        video_stream = youtube.streams.filter(
            progressive=False, resolution=resolution, subtype="mp4"
        ).first()
        if video_stream is not None:
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)
        else:
            video_stream = youtube.streams.filter(
                progressive=False, resolution=resolution
            ).first()
            if video_stream is None:
                print(f"Could not find a stream with resolution: {resolution}")
                print("Try one of these:")
                display_streams(youtube)
                sys.exit()
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)

def ffmpeg_downloader(youtube: YouTube, stream: Stream, target: Path) -> None:
    """
    Given a YouTube Stream object, finds the correct audio stream, downloads them both
    giving them a unique name, them uses ffmpeg to create a new file with the audio
    and video from the previously downloaded files. Then deletes the original adaptive
    streams, leaving the combination.

    :param YouTube youtube:
        A valid YouTube object
    :param Stream stream:
        A valid Stream object
    :param Path target:
        A valid Path object
    """
    audio_stream = (
        youtube.streams.filter(only_audio=True, subtype=stream.subtype)
        .order_by("abr")
        .desc()
        .first()
    )

    video_unique_name = unique_name(
        safe_filename(stream.title), stream.subtype, "video", target=target
    )
    audio_unique_name = unique_name(
        safe_filename(stream.title), stream.subtype, "audio", target=target
    )
    _download(stream=stream, target=str(target), filename=video_unique_name)
    _download(stream=audio_stream, target=str(target), filename=audio_unique_name)

    video_path = Path(target) / f"{video_unique_name}.{stream.subtype}"
    audio_path = Path(target) / f"{audio_unique_name}.{stream.subtype}"
    final_path = Path(target) / f"{safe_filename(stream.title)}.{stream.subtype}"

    subprocess.run( # nosec
        [
            "ffmpeg",
            "-i",
            f"{video_path}",
            "-i",
            f"{audio_path}",
            "-codec",
            "copy",
            f"{final_path}",
        ]
    )
    video_path.unlink()
    audio_path.unlink()

def download_by_itag(youtube: YouTube, itag: int, target: Optional[str] = None) -> None:
    """Start downloading a YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param int itag:
        YouTube format identifier code.
    :param str target:
        Target directory for download
    """
    stream = youtube.streams.get_by_itag(itag)
    if stream is None:
        print(f"Could not find a stream with itag: {itag}")
        print("Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(stream, target=target)
    except KeyboardInterrupt:
        sys.exit()

def download_by_resolution(
    youtube: YouTube, resolution: str, target: Optional[str] = None
) -> None:
    """Start downloading a YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str resolution:
        YouTube video resolution.
    :param str target:
        Target directory for download
    """
    # TODO(nficano): allow dash itags to be selected
    stream = youtube.streams.get_by_resolution(resolution)
    if stream is None:
        print(f"Could not find a stream with resolution: {resolution}")
        print("Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(stream, target=target)
    except KeyboardInterrupt:
        sys.exit()

def display_streams(youtube: YouTube) -> None:
    """Probe YouTube video and lists its available formats.

    :param YouTube youtube:
        A valid YouTube watch URL.

    """
    for stream in youtube.streams.all():
        print(stream)

def _print_available_captions(captions: CaptionQuery) -> None:
    print(f"Available caption codes are: {', '.join(c.code for c in captions.all())}")

def download_caption(
    youtube: YouTube, lang_code: Optional[str], target: Optional[str] = None
) -> None:
    """Download a caption for the YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str lang_code:
        Language code desired for caption file.
        Prints available codes if the value is None
        or the desired code is not available.
    :param str target:
        Target directory for download
    """
    if lang_code is None:
        _print_available_captions(youtube.captions)
        return

    caption = youtube.captions.get_by_language_code(lang_code=lang_code)
    if caption:
        downloaded_path = caption.download(title=youtube.title, output_path=target)
        print(f"Saved caption file to: {downloaded_path}")
    else:
        print(f"Unable to find caption with code: {lang_code}")
        _print_available_captions(youtube.captions)

def download_audio(
    youtube: YouTube, filetype: str, target: Optional[str] = None
) -> None:
    """
    Given a filetype, downloads the highest quality available audio stream for a
    YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str filetype:
        Desired file format to download.
    :param str target:
        Target directory for download
    """
    audio = (
        youtube.streams.filter(only_audio=True, subtype=filetype)
        .order_by("abr")
        .desc()
        .first()
    )

    if audio is None:
        print("No audio only stream found. Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(audio, target=target)
    except KeyboardInterrupt:
        sys.exit()

if __name__ == "__main__":
    main()
<commit_msg>Formatting<commit_after>#!/usr/bin/env python3
"""A simple command line application to download youtube videos."""

import argparse
import datetime as dt
import gzip
import json
import logging
import os
import shutil
import sys
import subprocess  # nosec
from io import BufferedWriter
from typing import Any, Optional, List
from pathlib import Path

from pytube import __version__, CaptionQuery, Stream, Playlist
from pytube import YouTube
from pytube.exceptions import PytubeError
from pytube.helpers import safe_filename

logger = logging.getLogger(__name__)

def main():
    """Command line application to download youtube videos."""
    # noinspection PyTypeChecker
    parser = argparse.ArgumentParser(description=main.__doc__)
    args = _parse_args(parser)
    logging.getLogger().setLevel(max(3 - args.verbosity, 0) * 10)

    if not args.url or "youtu" not in args.url:
        parser.print_help()
        sys.exit(1)

    if "/playlist" in args.url:
        print("Loading playlist...")
        playlist = Playlist(args.url)
        if not args.target:
            args.target = safe_filename(playlist.title())
        for youtube_video in playlist.videos:
            try:
                _perform_args_on_youtube(youtube_video, args)
            except PytubeError as e:
                print(f"There was an error with video: {youtube_video}")
                print(e)
    else:
        print("Loading video...")
        youtube = YouTube(args.url)
        _perform_args_on_youtube(youtube, args)

def _perform_args_on_youtube(youtube: YouTube, args: argparse.Namespace) -> None:
    if args.list:
        display_streams(youtube)
    if args.build_playback_report:
        build_playback_report(youtube)
    if args.itag:
        download_by_itag(youtube=youtube, itag=args.itag, target=args.target)
    if hasattr(args, "caption_code"):
        download_caption(
            youtube=youtube, lang_code=args.caption_code, target=args.target
        )
    if args.resolution:
        download_by_resolution(
            youtube=youtube, resolution=args.resolution, target=args.target
        )
    if args.audio:
        download_audio(youtube=youtube, filetype=args.audio, target=args.target)
    if args.ffmpeg:
        ffmpeg_process(youtube=youtube, resolution=args.ffmpeg, target=args.target)

def _parse_args(
    parser: argparse.ArgumentParser, args: Optional[List] = None
) -> argparse.Namespace:
    parser.add_argument("url", help="The YouTube /watch or /playlist url", nargs="?")
    parser.add_argument(
        "--version", action="version", version="%(prog)s " + __version__,
    )
    parser.add_argument(
        "--itag", type=int, help="The itag for the desired stream",
    )
    parser.add_argument(
        "-r", "--resolution", type=str, help="The resolution for the desired stream",
    )
    parser.add_argument(
        "-l",
        "--list",
        action="store_true",
        help=(
            "The list option causes pytube cli to return a list of streams "
            "available to download"
        ),
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        dest="verbosity",
        help="Verbosity level",
    )
    parser.add_argument(
        "--build-playback-report",
        action="store_true",
        help="Save the html and js to disk",
    )
    parser.add_argument(
        "-c",
        "--caption-code",
        type=str,
        default=argparse.SUPPRESS,
        nargs="?",
        help=(
            "Download srt captions for given language code. "
            "Prints available language codes if no argument given"
        ),
    )
    parser.add_argument(
        "-t",
        "--target",
        help=(
            "The output directory for the downloaded stream. "
            "Default is current working directory"
        ),
    )
    parser.add_argument(
        "-a",
        "--audio",
        const="mp4",
        nargs="?",
        help=(
            "Download the audio for a given URL at the highest bitrate available"
            "Defaults to mp4 format if none is specified"
        ),
    )
    parser.add_argument(
        "-f",
        "--ffmpeg",
        const="best",
        nargs="?",
        help=(
            "If no resolution provided, will download the highest quality video and\
            audio streams avaiable, will try to download mp4 but will then download\
            a webm if none are found"
        ),
    )

    return parser.parse_args(args)

def build_playback_report(youtube: YouTube) -> None:
    """Serialize the request data to json for offline debugging.

    :param YouTube youtube:
        A YouTube object.
    """
    ts = int(dt.datetime.utcnow().timestamp())
    fp = os.path.join(os.getcwd(), f"yt-video-{youtube.video_id}-{ts}.json.gz")

    js = youtube.js
    watch_html = youtube.watch_html
    vid_info = youtube.vid_info

    with gzip.open(fp, "wb") as fh:
        fh.write(
            json.dumps(
                {
                    "url": youtube.watch_url,
                    "js": js,
                    "watch_html": watch_html,
                    "video_info": vid_info,
                }
            ).encode("utf8"),
        )

def display_progress_bar(
    bytes_received: int, filesize: int, ch: str = "█", scale: float = 0.55
) -> None:
    """Display a simple, pretty progress bar.

    Example:
    ~~~~~~~~
    PSY - GANGNAM STYLE(강남스타일) MV.mp4
    ↳ |███████████████████████████████████████| 100.0%

    :param int bytes_received:
        The delta between the total file size (bytes) and bytes already
        written to disk.
    :param int filesize:
        File size of the media stream in bytes.
    :param str ch:
        Character to use for presenting progress segment.
    :param float scale:
        Scale multiplier to reduce progress bar size.

    """
    columns = shutil.get_terminal_size().columns
    max_width = int(columns * scale)

    filled = int(round(max_width * bytes_received / float(filesize)))
    remaining = max_width - filled
    progress_bar = ch * filled + " " * remaining
    percent = round(100.0 * bytes_received / float(filesize), 1)
    text = f" ↳ |{progress_bar}| {percent}%\r"
    sys.stdout.write(text)
    sys.stdout.flush()

# noinspection PyUnusedLocal
def on_progress(
    stream: Any, chunk: Any, file_handler: BufferedWriter, bytes_remaining: int
) -> None:
    filesize = stream.filesize
    bytes_received = filesize - bytes_remaining
    display_progress_bar(bytes_received, filesize)

def _download(
    stream: Stream, target: Optional[str] = None, filename: Optional[str] = None
) -> None:
    filesize_megabytes = stream.filesize // 1048576
    print(f"{stream.default_filename} | {filesize_megabytes} MB")
    stream.download(output_path=target, filename=filename)
    sys.stdout.write("\n")

def unique_name(base: str, subtype: str, video_audio: str, target: Path) -> str:
    """
    Given a base name, the file format, and the target directory, will generate
    a filename unique for that directory and file format.
    :param str base:
        The given base-name.
    :param str subtype:
        The filetype of the video which will be downloaded.
    :param Path target:
        Target directory for download.
    """
    counter = 0
    while True:
        name = f"{base}_{video_audio}_{counter}"
        unique = target / f"{name}.{subtype}"

        if not unique.exists():
            return str(name)
        counter += 1

def ffmpeg_process(
    youtube: YouTube, resolution: str, target: Optional[str] = None
) -> None:
    """
    Decides the correct video stream to download, then calls ffmpeg_downloader.

    :param YouTube youtube:
        A valid YouTube object.
    :param str resolution:
        YouTube video resolution.
    :param str target:
        Target directory for download
    """
    youtube.register_on_progress_callback(on_progress)
    if target is None:
        target = Path.cwd()
    else:
        target = Path(target)

    if resolution == "best":
        highest_quality = (
            youtube.streams.filter(progressive=False)
            .order_by("resolution")
            .desc()
            .first()
        )

        video_stream = (
            youtube.streams.filter(progressive=False, subtype="mp4")
            .order_by("resolution")
            .desc()
            .first()
        )

        if highest_quality.resolution == video_stream.resolution:
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)
        else:
            ffmpeg_downloader(youtube=youtube, stream=highest_quality, target=target)
    else:
        video_stream = youtube.streams.filter(
            progressive=False, resolution=resolution, subtype="mp4"
        ).first()
        if video_stream is not None:
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)
        else:
            video_stream = youtube.streams.filter(
                progressive=False, resolution=resolution
            ).first()
            if video_stream is None:
                print(f"Could not find a stream with resolution: {resolution}")
                print("Try one of these:")
                display_streams(youtube)
                sys.exit()
            ffmpeg_downloader(youtube=youtube, stream=video_stream, target=target)

def ffmpeg_downloader(youtube: YouTube, stream: Stream, target: Path) -> None:
    """
    Given a YouTube Stream object, finds the correct audio stream, downloads them both
    giving them a unique name, them uses ffmpeg to create a new file with the audio
    and video from the previously downloaded files. Then deletes the original adaptive
    streams, leaving the combination.

    :param YouTube youtube:
        A valid YouTube object
    :param Stream stream:
        A valid Stream object
    :param Path target:
        A valid Path object
    """
    audio_stream = (
        youtube.streams.filter(only_audio=True, subtype=stream.subtype)
        .order_by("abr")
        .desc()
        .first()
    )

    video_unique_name = unique_name(
        safe_filename(stream.title), stream.subtype, "video", target=target
    )
    audio_unique_name = unique_name(
        safe_filename(stream.title), stream.subtype, "audio", target=target
    )
    _download(stream=stream, target=str(target), filename=video_unique_name)
    _download(stream=audio_stream, target=str(target), filename=audio_unique_name)

    video_path = Path(target) / f"{video_unique_name}.{stream.subtype}"
    audio_path = Path(target) / f"{audio_unique_name}.{stream.subtype}"
    final_path = Path(target) / f"{safe_filename(stream.title)}.{stream.subtype}"

    subprocess.run(  # nosec
        [
            "ffmpeg",
            "-i",
            f"{video_path}",
            "-i",
            f"{audio_path}",
            "-codec",
            "copy",
            f"{final_path}",
        ]
    )
    video_path.unlink()
    audio_path.unlink()

def download_by_itag(youtube: YouTube, itag: int, target: Optional[str] = None) -> None:
    """Start downloading a YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param int itag:
        YouTube format identifier code.
    :param str target:
        Target directory for download
    """
    stream = youtube.streams.get_by_itag(itag)
    if stream is None:
        print(f"Could not find a stream with itag: {itag}")
        print("Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(stream, target=target)
    except KeyboardInterrupt:
        sys.exit()

def download_by_resolution(
    youtube: YouTube, resolution: str, target: Optional[str] = None
) -> None:
    """Start downloading a YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str resolution:
        YouTube video resolution.
    :param str target:
        Target directory for download
    """
    # TODO(nficano): allow dash itags to be selected
    stream = youtube.streams.get_by_resolution(resolution)
    if stream is None:
        print(f"Could not find a stream with resolution: {resolution}")
        print("Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(stream, target=target)
    except KeyboardInterrupt:
        sys.exit()

def display_streams(youtube: YouTube) -> None:
    """Probe YouTube video and lists its available formats.

    :param YouTube youtube:
        A valid YouTube watch URL.

    """
    for stream in youtube.streams.all():
        print(stream)

def _print_available_captions(captions: CaptionQuery) -> None:
    print(f"Available caption codes are: {', '.join(c.code for c in captions.all())}")

def download_caption(
    youtube: YouTube, lang_code: Optional[str], target: Optional[str] = None
) -> None:
    """Download a caption for the YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str lang_code:
        Language code desired for caption file.
        Prints available codes if the value is None
        or the desired code is not available.
    :param str target:
        Target directory for download
    """
    if lang_code is None:
        _print_available_captions(youtube.captions)
        return

    caption = youtube.captions.get_by_language_code(lang_code=lang_code)
    if caption:
        downloaded_path = caption.download(title=youtube.title, output_path=target)
        print(f"Saved caption file to: {downloaded_path}")
    else:
        print(f"Unable to find caption with code: {lang_code}")
        _print_available_captions(youtube.captions)

def download_audio(
    youtube: YouTube, filetype: str, target: Optional[str] = None
) -> None:
    """
    Given a filetype, downloads the highest quality available audio stream for a
    YouTube video.

    :param YouTube youtube:
        A valid YouTube object.
    :param str filetype:
        Desired file format to download.
    :param str target:
        Target directory for download
    """
    audio = (
        youtube.streams.filter(only_audio=True, subtype=filetype)
        .order_by("abr")
        .desc()
        .first()
    )

    if audio is None:
        print("No audio only stream found. Try one of these:")
        display_streams(youtube)
        sys.exit()

    youtube.register_on_progress_callback(on_progress)

    try:
        _download(audio, target=target)
    except KeyboardInterrupt:
        sys.exit()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#
# Copyright 2016 Michael Sparks
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function
from __future__ import absolute_import

import os
import pprint
import time

import pyxie.codegen.profiles as profiles
import pyxie.codegen.simple_cpp

from pyxie.parsing.lexer import build_lexer
from pyxie.parsing.grammar import parse

from pyxie.model.pynode import jdump
from pyxie.model.transform import ast_to_cst

from pyxie.codegen.clib import files as clib_files
from pyxie.codegen.simple_cpp import C_Program, source, reset_parser

testdir = "test-data"
testprogs_dir = os.path.join(testdir, "progs")

def remove_directory(build_dir):
    for filename in os.listdir(build_dir):
        try:
            os.unlink(os.path.join(build_dir, filename))
        except OSError as e :
            if e.errno == 21:
                remove_directory(os.path.join(build_dir, filename))
    os.rmdir(build_dir)

def get_test_programs(program_suffix):
    testprogs = [x for x in os.listdir(testprogs_dir) ]
    testprogs = [x for x in testprogs if x.endswith(program_suffix) ]
    return testprogs

def get_build_environment(filename, result_filename):
    rootdir = os.getcwd()

    lastslash_pos = filename.rfind("/")
    if lastslash_pos != -1:
        base_dir = filename[:lastslash_pos]
        base_filename = filename[lastslash_pos+1:]
    else:
        base_dir = "." # just a filename
        base_filename = filename

    cname = base_filename[:base_filename.rfind(".")]

    if not result_filename:
        result_filename = os.path.join(base_dir, cname)

    return {
            "rootdir" :rootdir,
            "base_dir" : base_dir,
            "base_filename":base_filename,
            "cname" : cname,
            "result_filename" : result_filename,
           }

def parse_file(somefile):
    lexer = build_lexer()
    reset_parser()
    data = open(somefile).read() + "\n#\n"
    AST = parse(data, lexer)
    AST.includes = lexer.includes
    print("AST includes", AST.includes)
    print("PRINT AST ----------------------------------")
    print(AST)
    print("---------------------------------- PRINT AST")
    return AST

def analyse_file(filename):
    try:
        AST = parse_file(filename)
        print("HERE 3------------")
        pprint.pprint(AST.__json__())
        print("HERE 4------------")
        AST.analyse()
        print("HERE 5------------")
        pprint.pprint(AST.__info__())
        print("HERE 6------------")
    except:
        from pyxie.parsing.context import Context
        for cid in Context.contexts:
            context = Context.contexts[cid]
            print("CONTEXT",context.names)
        raise

def generate_code(cname, AST, profile, debug=False):
    CST = ast_to_cst(cname, AST)
    if debug:
        print("CONCRETE C SYNTAX TREE: - - - - - - - - - - - - - - - - - - - -")
        print(pprint.pformat(CST))
        print("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -")
    program = C_Program.fromjson(CST)
    program.generate(profile)
    return pyxie.codegen.simple_cpp.source[:]

def codegen_phase(filename, result_filename, profile):
    build_env = get_build_environment(filename, result_filename)

    cname = build_env["cname"]

    AST = parse_file(filename)
    AST.analyse()
    print("_______________________________________________________________")
    print("GENERATING CODE")
    print()
    c_code = generate_code(cname, AST, profile, debug=True)  # Need the C NAME TO DO THIS
    print("_______________________________________________________________")
    print("Generated Program")
    print("\n".join(c_code))
    print("_______________________________________________________________")
    return c_code

def build_program(source, work_dir, name, profile):
    print("_______________________________________________________________")
    print("BUILDING", work_dir)
    print()
    print("Program to build:")
    pprint.pprint(source, width=200)
    print()

    extension = profiles.mainfile_extensions.get(profile,"c")

    source_filename = os.path.join(work_dir,name+"." +extension)

#    f = open(os.path.join(work_dir,name+"." +extension), "w")
    f = open(source_filename, "w")
    for line in source:
        f.write(line)
        f.write("\n")
    f.close()

    if os.path.exists("/usr/bin/indent"):
        try:
            print("TIDYING")
            print("/usr/bin/indent %s" % source_filename)
            os.system("/usr/bin/indent -npcs -brf -br -npsl -l120 -i4 -nut %s" % source_filename)
        except Exception as e:
            print("TIDYING Failed - continuing anyway")
            print("TIDYING error: %s %s" % (str(e), repr(e)))

    try:
        makefile_tmpl = profiles.makefile_templates[profile]
    except KeyError:
        makefile_tmpl = profiles.makefile_templates["default"]

    makefile = makefile_tmpl % {"filename": name }
    f = open(os.path.join(work_dir,"Makefile"), "w")
    f.write(makefile)
    f.close()

    exclusions = profiles.clib_exclusions.get(profile, [])
    for filename in clib_files:
        if filename.endswith("_test.cpp"):
            # Skip main programs to avoid confusing profiles
            continue
        if filename in exclusions:
            print("Skipping", filename, "for profile", profile)
            continue

        f = open( os.path.join(work_dir,filename), "w")
        f.write(clib_files[filename])
        f.close()

    os.chdir(work_dir)
    os.system("make")
    print()
    print("Done!")
    print()

# Next function is called by compile_file, but could be called independently
# This is why both do "build_env", rather that expect the caller to
def compile_file(filename, profile, result_filename=None):

    build_env = get_build_environment(filename, result_filename)

    rootdir = build_env["rootdir"]
    base_dir = build_env["base_dir"]
    base_filename = build_env["base_filename"]
    cname = build_env["cname"]
    result_filename = build_env["result_filename"]

    c_code  = codegen_phase(filename, result_filename, profile)

    print("COMPILING", filename)
    print("IN", base_dir)
    print("SOURCEFILE", base_filename)
    print("cname", cname)
    print("result_filename", result_filename)

    build_dir = os.path.join(base_dir, "build-"+str(int(time.time())))
    try:
        os.mkdir(build_dir)
    except OSError as e:
       if e.errno != 17: # Directory exists
           raise

    build_program(c_code, build_dir, cname, profile)

    os.chdir(rootdir)

    actual_result_file = (profiles.result_file[profile])(build_dir, cname)

    result_filename = (profiles.modify_result_file[profile])(result_filename)

    print("BUILD DIR", build_dir)
    print("RESULT FILE", actual_result_file)
    print("DEST FILE", result_filename)
    print("CWD", os.getcwd())

    os.rename(actual_result_file,result_filename)
    clean = False
    if clean:
        remove_directory(build_dir)
        if base_dir == ".":
            os.unlink("parsetab.py")
            os.unlink("parser.out")
    return

def parse_testfile(testprogs_dir, testfile, debug=False):
    print("_______________________________________________________________")
    print("PARSING", os.path.join(testprogs_dir,testfile))
    print()
    AST = parse_file(os.path.join(testprogs_dir,testfile))
    if debug:
        pprint.pprint(AST)
        JAST = jdump(AST)
        pprint.pprint(JAST)
    return AST

def compile_testfile(testprogs_dir, testfile, profile):
    # The compiled c name is the filename with the suffix removed
    p = testfile.rfind(".")
    cname = testfile[:p]

    AST = parse_testfile(testprogs_dir, testfile, debug=True)

    print("_______________________________________________________________")
    print("ANALYSING", os.path.join(testprogs_dir,testfile))
    print()
    AST.analyse()

    print("_______________________________________________________________")
    print("COMPILING", os.path.join(testprogs_dir,testfile))
    print()

    c_code = generate_code(cname, AST, profile, debug=True)

    # Create Work directory
    allresults_dir = os.path.join(testdir, "genprogs")
    thisresult_dir = os.path.join(allresults_dir, testfile)

    try:
        os.mkdir(allresults_dir)
    except OSError as e:
        pass # Can raise an exception if the directory exists, which is fine

    try:
        os.mkdir(thisresult_dir)
    except OSError as e:
       if e.errno != 17: # Directory exists
           raise

    build_program(c_code, thisresult_dir, cname, profile)

def parsing_tests():
    # Run parsing tests. These are in the "progs" test directory, and are in
    # filenames ending ".p"

    rootdir = os.getcwd()
    testprogs = get_test_programs(".p")

    for testfile in testprogs:
        AST = parse_testfile(testprogs_dir, testfile)
        pprint.pprint(AST)

        JAST = jdump(AST)
        pprint.pprint(JAST)

        os.chdir(rootdir)

def compilation_tests(profile):
    # ast_to_cst
    # Compilation Tests
    rootdir = os.getcwd()
    testprogs = get_test_programs(".pyxie")
    print("TEST PROGS", testprogs)

    for testfile in testprogs:
        build_fail = True
        try:
            compile_testfile(testprogs_dir, testfile, profile)
            build_fail = False
        except:
            if testfile.endswith("shouldfail.pyxie"):
                print("AS EXPECTED, COMPILE FAILED FOR", testfile)
            else:
                print("UNEXPECTED FAILURE FOR FILE", testfile)
                raise

        if not build_fail and testfile.endswith("shouldfail.pyxie"):
            print("UNEXPECTED BUILD SUCCESS FOR FILE", testfile)
            raise Exception("UNEXPECTED BUILD SUCCESS FOR FILE %s" % testfile )

        os.chdir(rootdir)

    print("COMPILING DONE", testprogs)
<commit_msg>Add mechanism for using a Makefile.in file to override arduino build board tag<commit_after>#
# Copyright 2016 Michael Sparks
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function
from __future__ import absolute_import

import os
import pprint
import time

import pyxie.codegen.profiles as profiles
import pyxie.codegen.simple_cpp

from pyxie.parsing.lexer import build_lexer
from pyxie.parsing.grammar import parse

from pyxie.model.pynode import jdump
from pyxie.model.transform import ast_to_cst

from pyxie.codegen.clib import files as clib_files
from pyxie.codegen.simple_cpp import C_Program, source, reset_parser

testdir = "test-data"
testprogs_dir = os.path.join(testdir, "progs")

def copy_file(source, dest):
    f = open(source, "rb")
    g = open(dest, "wb")
    g.write(f.read())
    f.close()
    g.close()

def remove_directory(build_dir):
    for filename in os.listdir(build_dir):
        try:
            os.unlink(os.path.join(build_dir, filename))
        except OSError as e :
            if e.errno == 21:
                remove_directory(os.path.join(build_dir, filename))
    os.rmdir(build_dir)

def get_test_programs(program_suffix):
    testprogs = [x for x in os.listdir(testprogs_dir) ]
    testprogs = [x for x in testprogs if x.endswith(program_suffix) ]
    return testprogs

def get_build_environment(filename, result_filename):
    rootdir = os.getcwd()

    lastslash_pos = filename.rfind("/")
    if lastslash_pos != -1:
        base_dir = filename[:lastslash_pos]
        base_filename = filename[lastslash_pos+1:]
    else:
        base_dir = "." # just a filename
        base_filename = filename

    cname = base_filename[:base_filename.rfind(".")]

    if not result_filename:
        result_filename = os.path.join(base_dir, cname)

    return {
            "rootdir" :rootdir,
            "base_dir" : base_dir,
            "base_filename":base_filename,
            "cname" : cname,
            "result_filename" : result_filename,
           }

def parse_file(somefile):
    lexer = build_lexer()
    reset_parser()
    data = open(somefile).read() + "\n#\n"
    AST = parse(data, lexer)
    AST.includes = lexer.includes
    print("AST includes", AST.includes)
    print("PRINT AST ----------------------------------")
    print(AST)
    print("---------------------------------- PRINT AST")
    return AST

def analyse_file(filename):
    try:
        AST = parse_file(filename)
        print("HERE 3------------")
        pprint.pprint(AST.__json__())
        print("HERE 4------------")
        AST.analyse()
        print("HERE 5------------")
        pprint.pprint(AST.__info__())
        print("HERE 6------------")
    except:
        from pyxie.parsing.context import Context
        for cid in Context.contexts:
            context = Context.contexts[cid]
            print("CONTEXT",context.names)
        raise

def generate_code(cname, AST, profile, debug=False):
    CST = ast_to_cst(cname, AST)
    if debug:
        print("CONCRETE C SYNTAX TREE: - - - - - - - - - - - - - - - - - - - -")
        print(pprint.pformat(CST))
        print("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -")
    program = C_Program.fromjson(CST)
    program.generate(profile)
    return pyxie.codegen.simple_cpp.source[:]

def codegen_phase(filename, result_filename, profile):
    build_env = get_build_environment(filename, result_filename)

    cname = build_env["cname"]

    AST = parse_file(filename)
    AST.analyse()
    print("_______________________________________________________________")
    print("GENERATING CODE")
    print()
    c_code = generate_code(cname, AST, profile, debug=True)  # Need the C NAME TO DO THIS
    print("_______________________________________________________________")
    print("Generated Program")
    print("\n".join(c_code))
    print("_______________________________________________________________")
    return c_code

def build_program(source, work_dir, name, profile):
    print("_______________________________________________________________")
    print("BUILDING", work_dir)
    print()
    print("Program to build:")
    pprint.pprint(source, width=200)
    print()

    extension = profiles.mainfile_extensions.get(profile,"c")

    source_filename = os.path.join(work_dir,name+"." +extension)

#    f = open(os.path.join(work_dir,name+"." +extension), "w")
    f = open(source_filename, "w")
    for line in source:
        f.write(line)
        f.write("\n")
    f.close()

    if os.path.exists("/usr/bin/indent"):
        try:
            print("TIDYING")
            print("/usr/bin/indent %s" % source_filename)
            os.system("/usr/bin/indent -npcs -brf -br -npsl -l120 -i4 -nut %s" % source_filename)
        except Exception as e:
            print("TIDYING Failed - continuing anyway")
            print("TIDYING error: %s %s" % (str(e), repr(e)))

    try:
        makefile_tmpl = profiles.makefile_templates[profile]
    except KeyError:
        makefile_tmpl = profiles.makefile_templates["default"]

    makefile = makefile_tmpl % {"filename": name }

    # This is hideously inefficient. It's simple though
    if os.path.exists(os.path.join(work_dir,"Makefile.in")):
        # MAKEFILE.IN EXISTS
        # Parse it into a dictionary
        f = open(os.path.join(work_dir,"Makefile.in"))
        lines = f.readlines()
        f.close()
        r = {}
        for line in lines:
            p = line.find("=")
            key, value = line[:p].strip(), line[p+1:].strip()
            r[key] = value

        # Loop through default makefile and replace values in it with overrides
        makefile_ = []
        makefile = makefile.split("\n")
        for line in makefile:
            if "=" in line:
                p = line.find("=")
                key, value = line[:p].strip(), line[p+1:].strip()
                if key in r:
                    makefile_.append( key + " = " + r[key])
                else:
                    makefile_.append(line)
            else:
                    makefile_.append(line)

        makefile = "\n".join(makefile_)

    f = open(os.path.join(work_dir,"Makefile"), "w")
    f.write(makefile)
    f.close()

    exclusions = profiles.clib_exclusions.get(profile, [])
    for filename in clib_files:
        if filename.endswith("_test.cpp"):
            # Skip main programs to avoid confusing profiles
            continue
        if filename in exclusions:
            print("Skipping", filename, "for profile", profile)
            continue

        f = open( os.path.join(work_dir,filename), "w")
        f.write(clib_files[filename])
        f.close()

    os.chdir(work_dir)
    os.system("make")
    print()
    print("Done!")
    print()

# Next function is called by compile_file, but could be called independently
# This is why both do "build_env", rather that expect the caller to
def compile_file(filename, profile, result_filename=None):

    build_env = get_build_environment(filename, result_filename)

    rootdir = build_env["rootdir"]
    base_dir = build_env["base_dir"]
    base_filename = build_env["base_filename"]
    cname = build_env["cname"]
    result_filename = build_env["result_filename"]

    c_code  = codegen_phase(filename, result_filename, profile)

    print("COMPILING", filename)
    print("IN", base_dir)
    print("SOURCEFILE", base_filename)
    print("cname", cname)
    print("result_filename", result_filename)

    build_dir = os.path.join(base_dir, "build-"+str(int(time.time())))
    try:
        os.mkdir(build_dir)
    except OSError as e:
       if e.errno != 17: # Directory exists
           raise

    if (os.path.exists(result_filename+".Makefile.in")):
        # USER OPTIONS FOR COMPILING DETECTED - copy into build directory
        copy_file(result_filename+".Makefile.in", os.path.join(build_dir, "Makefile.in"))

    build_program(c_code, build_dir, cname, profile)

    os.chdir(rootdir)

    actual_result_file = (profiles.result_file[profile])(build_dir, cname)

    result_filename = (profiles.modify_result_file[profile])(result_filename)

    print("BUILD DIR", build_dir)
    print("RESULT FILE", actual_result_file)
    print("DEST FILE", result_filename)
    print("CWD", os.getcwd())

    os.rename(actual_result_file,result_filename)
    clean = False
    if clean:
        remove_directory(build_dir)
        if base_dir == ".":
            os.unlink("parsetab.py")
            os.unlink("parser.out")
    return

def parse_testfile(testprogs_dir, testfile, debug=False):
    print("_______________________________________________________________")
    print("PARSING", os.path.join(testprogs_dir,testfile))
    print()
    AST = parse_file(os.path.join(testprogs_dir,testfile))
    if debug:
        pprint.pprint(AST)
        JAST = jdump(AST)
        pprint.pprint(JAST)
    return AST

def compile_testfile(testprogs_dir, testfile, profile):
    # The compiled c name is the filename with the suffix removed
    p = testfile.rfind(".")
    cname = testfile[:p]

    AST = parse_testfile(testprogs_dir, testfile, debug=True)

    print("_______________________________________________________________")
    print("ANALYSING", os.path.join(testprogs_dir,testfile))
    print()
    AST.analyse()

    print("_______________________________________________________________")
    print("COMPILING", os.path.join(testprogs_dir,testfile))
    print()

    c_code = generate_code(cname, AST, profile, debug=True)

    # Create Work directory
    allresults_dir = os.path.join(testdir, "genprogs")
    thisresult_dir = os.path.join(allresults_dir, testfile)

    try:
        os.mkdir(allresults_dir)
    except OSError as e:
        pass # Can raise an exception if the directory exists, which is fine

    try:
        os.mkdir(thisresult_dir)
    except OSError as e:
       if e.errno != 17: # Directory exists
           raise

    build_program(c_code, thisresult_dir, cname, profile)

def parsing_tests():
    # Run parsing tests. These are in the "progs" test directory, and are in
    # filenames ending ".p"

    rootdir = os.getcwd()
    testprogs = get_test_programs(".p")

    for testfile in testprogs:
        AST = parse_testfile(testprogs_dir, testfile)
        pprint.pprint(AST)

        JAST = jdump(AST)
        pprint.pprint(JAST)

        os.chdir(rootdir)

def compilation_tests(profile):
    # ast_to_cst
    # Compilation Tests
    rootdir = os.getcwd()
    testprogs = get_test_programs(".pyxie")
    print("TEST PROGS", testprogs)

    for testfile in testprogs:
        build_fail = True
        try:
            compile_testfile(testprogs_dir, testfile, profile)
            build_fail = False
        except:
            if testfile.endswith("shouldfail.pyxie"):
                print("AS EXPECTED, COMPILE FAILED FOR", testfile)
            else:
                print("UNEXPECTED FAILURE FOR FILE", testfile)
                raise

        if not build_fail and testfile.endswith("shouldfail.pyxie"):
            print("UNEXPECTED BUILD SUCCESS FOR FILE", testfile)
            raise Exception("UNEXPECTED BUILD SUCCESS FOR FILE %s" % testfile )

        os.chdir(rootdir)

    print("COMPILING DONE", testprogs)
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import sys, socket, time

sleep_time = 1
file_to_save = ""

class mysocket:

	def __init__(self,ip = None ,port = None, sock=None,timeout=0.1):
		if sock is None:
			self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		else:
			self.sock = sock
		self.sock.settimeout(timeout)
		self.connection = False
		if type(ip) == str and type(port) == int:
			self.connection = self.connect(ip,port)

	def connect(self, host, port):
		try:
			self.sock.connect((host, port))
			return True
		except:
			return False

	def send(self, msg = "", n = None):
		if type(n) == int:
			msg = msg[:n]
		try :
			if self.sock.send(msg) == 0:
				return False
			return True
		except:
			return False

	def receive(self):
		msg = ""
		c = self.get_one_char()
		while (c != "\n") \
		and (c != False):
			msg = msg + c
			c = self.get_one_char()
		if msg == "":
			return False
		return msg

	def get_data(self):
		msg = ""
		tm = self.receive()
		while tm :
			msg = msg + tm + "\n"
			tm = self.receive()
		if msg == "":
			return False
		return msg

	def get_one_char(self):
		try:
			c = self.sock.recv(1)
		except:
			c = False
		return c

	def comunicate(self,msg=None):
		try:
			if type(msg) == str:
				connection.send(msg+"\n")
			return connection.get_data()
		except:
			return False

def nums_only(string):
	nstr = ""
	for c in string:
		for n in "0123456789":
			if n == c:
				nstr = nstr + n
				break
	return nstr

def extract_numbers(string):
	if type(string) != str:
		return False
	t = string.split(" ")
	if t[0] != "Nope,":
		return False
	sys.stdout.flush()
	return nums_only(t[8])

def append_to_file(toappend):
	f = open(file_to_save, "a")
	f.write(toappend+"\n")
	f.close()

if __name__ == "__main__":

	try:
		connection = mysocket(sys.argv[1],int(sys.argv[2]))
	except:
		connection = mysocket('127.0.0.1',8888)

	if connection.connection:
		#print "timeout : '" + str(connection.sock.gettimeout()) + "'"
		#print connection.comunicate()
		connection.comunicate()
		while True:
			rep = extract_numbers(connection.comunicate(""))
			if rep == False:
				print "error comunicating"
				break
			append_to_file(rep)
			print  rep
			time.sleep(sleep_time)

	else:
		print "no connection, start server first"
<commit_msg>updated<commit_after>#!/usr/bin/env python2

import sys, socket, time

sleep_time = 0
file_to_save = "file.txt"

class mysocket:

	def __init__(self,ip = None ,port = None, sock=None,timeout=1):
		if sock is None:
			self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		else:
			self.sock = sock
		self.sock.settimeout(timeout)
		self.connection = False
		if type(ip) == str and type(port) == int:
			self.connection = self.connect(ip,port)

	def connect(self, host, port):
		try:
			self.sock.connect((host, port))
			return True
		except:
			return False

	def send(self, msg = "", n = None):
		if type(n) == int:
			msg = msg[:n]
		try :
			if self.sock.send(msg) == 0:
				return False
			return True
		except:
			return False

	def receive(self):
		msg = ""
		c = self.get_one_char()
		while (c != "\n") \
		and (c != False):
			msg = msg + c
			c = self.get_one_char()
		if msg == "":
			return False
		return msg

	def get_data(self):
		msg = ""
		tm = self.receive()
		while tm :
			msg = msg + tm + "\n"
			tm = self.receive()
		if msg == "":
			return False
		return msg

	def get_one_char(self):
		try:
			c = self.sock.recv(1)
		except:
			c = False
		return c

	def comunicate(self,msg=None):
		try:
			if type(msg) == str:
				connection.send(msg+"\n")
			return connection.get_data()
		except:
			return False

def nums_only(string):
	nstr = ""
	for c in string:
		for n in "0123456789":
			if n == c:
				nstr = nstr + n
				break
	return nstr

def extract_numbers(string):
	if type(string) != str:
		return False
	t = string.split(" ")
	if t[0] != "Nope,":
		return False
	sys.stdout.flush()
	return nums_only(t[8])

def append_to_file(toappend):
	f = open(file_to_save, "a")
	f.write(toappend+"\n")
	f.close()

def set_keepalive_linux(sock, after_idle_sec=1, interval_sec=3, max_fails=5):
    """Set TCP keepalive on an open socket.

    It activates after 1 second (after_idle_sec) of idleness,
    then sends a keepalive ping once every 3 seconds (interval_sec),
    and closes the connection after 5 failed ping (max_fails), or 15 seconds
    """
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec)
    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec)
    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails)

if __name__ == "__main__":

	try:
		connection = mysocket(sys.argv[1],int(sys.argv[2]))
	except:
		connection = mysocket('127.0.0.1',8888)

	if connection.connection:
		set_keepalive_linux(connection.sock)
		#print "timeout : '" + str(connection.sock.gettimeout()) + "'"
		#print connection.comunicate()
		connection.comunicate()
		while True:
			rep = extract_numbers(connection.comunicate(""))
			if rep == False:
				print "error comunicating"
				break
			append_to_file(rep)
			print  rep
			time.sleep(sleep_time)

	else:
		print "no connection, start server first"
<|endoftext|>


<commit_before># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg
from oslo_db import options
from oslo_log import log

from nova import debugger
from nova import paths
from nova import rpc
from nova import version

CONF = cfg.CONF

_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nova.sqlite')

_DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
                       'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
                       'oslo.messaging=INFO', 'iso8601=WARN',
                       'requests.packages.urllib3.connectionpool=WARN',
                       'urllib3.connectionpool=WARN', 'websocket=WARN',
                       'keystonemiddleware=WARN', 'routes.middleware=WARN',
                       'stevedore=WARN', 'glanceclient=WARN']

_DEFAULT_LOGGING_CONTEXT_FORMAT = ('%(asctime)s.%(msecs)03d %(process)d '
                                   '%(levelname)s %(name)s [%(request_id)s '
                                   '%(user_identity)s] %(instance)s'
                                   '%(message)s')

def parse_args(argv, default_config_files=None):
    log.set_defaults(_DEFAULT_LOGGING_CONTEXT_FORMAT, _DEFAULT_LOG_LEVELS)
    log.register_options(CONF)
    options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION,
                         sqlite_db='nova.sqlite')
    rpc.set_defaults(control_exchange='nova')
    debugger.register_cli_opts()
    CONF(argv[1:],
         project='nova',
         version=version.version_string(),
         default_config_files=default_config_files)
    rpc.init(CONF)
<commit_msg>Disable oslo_messaging debug logging<commit_after># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_config import cfg
from oslo_db import options
from oslo_log import log

from nova import debugger
from nova import paths
from nova import rpc
from nova import version

CONF = cfg.CONF

_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nova.sqlite')

_DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
                       'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
                       'oslo_messaging=INFO', 'iso8601=WARN',
                       'requests.packages.urllib3.connectionpool=WARN',
                       'urllib3.connectionpool=WARN', 'websocket=WARN',
                       'keystonemiddleware=WARN', 'routes.middleware=WARN',
                       'stevedore=WARN', 'glanceclient=WARN']

_DEFAULT_LOGGING_CONTEXT_FORMAT = ('%(asctime)s.%(msecs)03d %(process)d '
                                   '%(levelname)s %(name)s [%(request_id)s '
                                   '%(user_identity)s] %(instance)s'
                                   '%(message)s')

def parse_args(argv, default_config_files=None):
    log.set_defaults(_DEFAULT_LOGGING_CONTEXT_FORMAT, _DEFAULT_LOG_LEVELS)
    log.register_options(CONF)
    options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION,
                         sqlite_db='nova.sqlite')
    rpc.set_defaults(control_exchange='nova')
    debugger.register_cli_opts()
    CONF(argv[1:],
         project='nova',
         version=version.version_string(),
         default_config_files=default_config_files)
    rpc.init(CONF)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    finances.models
    ~~~~~~~~~~~~~~

    Has the content related to models for finances.

    :copyright: (c) 2012 by Arruda.
"""

import datetime
from decimal import Decimal

from django.db import models
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices

from utils.abs_models import Abs_Named_Model

class FinancialTransaction(Abs_Named_Model):
    """Represents a financial transaction:

        name: the name of this transaction, something that explains where the money is comming or going to.
        project: the project that this transaction is related to.
        transaction_type: if its a debit or credit.

        --
        start_date: The date of the first parcel.
        end_date: The date of the last parcel.
        total: The sum of all parcel's values
    """

    TRANSACTION_TYPE = Choices(
                            ('d', 'debit', _('Debit')),
                            ('c', 'credit', _('Credit')),
                            )

    project = models.ForeignKey('projects.Project')

    transaction_type = models.CharField(_('Transaction Type'), choices=TRANSACTION_TYPE, default=TRANSACTION_TYPE.debit)

    class Meta:
        app_label = 'finances'

class TransactionParcel(models.Model):
    """
    It's a parcel from a transaction.
    """

    transaction = models.ForeignKey(FinancialTransaction,related_name='parcels')

    value = models.DecimalField(_("Value"),max_digits=12, decimal_places=2,default=Decimal("0"))

    date = models.DateField(_("Date"), default=datetime.date.today)

    class Meta:
        app_label = 'finances'

    def __unicode__(self):
        return "% - %  - %s " % (self.transaction, self.date, self.value)
    <commit_msg>adding type aware and proxies fir debit and credit transactions<commit_after># -*- coding: utf-8 -*-
"""
    finances.models
    ~~~~~~~~~~~~~~

    Has the content related to models for finances.

    :copyright: (c) 2012 by Arruda.
"""

import datetime
from decimal import Decimal

from django.db import models
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices

from utils.abs_models import Abs_Named_Model

class TransactionTypeAwareManager(models.Manager):
    """
    Manager for Transaction that knows how to deal with the credit and debit proxies
    """
    def __init__(self, type=None, *args, **kwargs):
        super(TransactionTypeAwareManager, self).__init__(*args, **kwargs)
        self.type = type

    def get_query_set(self):
        qq = super(TransactionTypeAwareManager, self).get_query_set()
        if self.type:
            qq = qq.filter(transaction_type=self.type)

        return qq

class FinancialTransaction(Abs_Named_Model):
    """Represents a financial transaction:

        name: the name of this transaction, something that explains where the money is comming or going to.
        project: the project that this transaction is related to.
        transaction_type: if its a debit or credit.

        --
        start_date: The date of the first parcel.
        end_date: The date of the last parcel.
        total: The sum of all parcel's values
    """

    TRANSACTION_TYPE = Choices(
                            ('d', 'debit', _('Debit')),
                            ('c', 'credit', _('Credit')),
                            )

    project = models.ForeignKey('projects.Project')

    transaction_type = models.CharField(_('Transaction Type'), choices=TRANSACTION_TYPE, default=TRANSACTION_TYPE.debit)

    objects = TransactionTypeAwareManager()
    class Meta:
        app_label = 'finances'

class DebitTransaction(FinancialTransaction):

    class Meta:
        app_label = 'finances'
        prox = True

    objects = TransactionTypeAwareManager(type=FinancialTransaction.TRANSACTION_TYPE.debit)

    def save(self, *args, **kwargs):
        self.transaction_type = FinancialTransaction.TRANSACTION_TYPE.debit
        return super(CreditTransaction, self).save(*args, **kwargs)

class CreditTransaction(FinancialTransaction):

    class Meta:
        app_label = 'finances'
        prox = True

    objects = TransactionTypeAwareManager(type=FinancialTransaction.TRANSACTION_TYPE.credit)

    def save(self, *args, **kwargs):
        self.transaction_type = FinancialTransaction.TRANSACTION_TYPE.credit
        return super(CreditTransaction, self).save(*args, **kwargs)

class TransactionParcel(models.Model):
    """
    It's a parcel from a transaction.
    """

    transaction = models.ForeignKey(FinancialTransaction,related_name='parcels')

    value = models.DecimalField(_("Value"),max_digits=12, decimal_places=2,default=Decimal("0"))

    date = models.DateField(_("Date"), default=datetime.date.today)

    class Meta:
        app_label = 'finances'
        ordering = ['date',]

    def __unicode__(self):
        return "% - %  - %s " % (self.transaction, self.date, self.value)
    <|endoftext|>


<commit_before># -*- mode: Python;-*-

import sys
import operator

# Cost is basically: was there a match or not.
# The other numbers are cumulative costs and matches...

def lowest_cost_action(ic, dc, sc, im, dm, sm, cost):
    """Given the following values, choose the action (insertion, deletion,
    or substitution), that results in the lowest cost (ties are broken using
    the 'match' score).  This is used within the dynamic programming algorithm.
    * ic - insertion cost
    * dc - deletion cost
    * sc - substitution cost
    * im - insertion match (score)
    * dm - deletion match (score)
    * sm - substitution match (score)
    """
    best_action = None
    best_match_count = -1
    min_cost = min(ic, dc, sc)
    if min_cost == sc and cost == 0:
        best_action = 'equal'
        best_match_count = sm
    elif min_cost == sc and cost == 1:
        best_action = 'replace'
        best_match_count = sm
    elif min_cost == ic and im > best_match_count:
        best_action = 'insert'
        best_match_count = im
    elif min_cost == dc and dm > best_match_count:
        best_action = 'delete'
        best_match_count = dm
    return best_action

def highest_match_action(ic, dc, sc, im, dm, sm, cost):
    """Given the following values, choose the action (insertion, deletion, or
    substitution), that results in the highest match score (ties are broken
    using the distance values).  This is used within the dynamic programming
    algorithm.
    * ic - insertion cost
    * dc - deletion cost
    * sc - substitution cost
    * im - insertion match (score)
    * dm - deletion match (score)
    * sm - substitution match (score)
    """
    best_action = None
    lowest_cost = float("inf")
    max_match = max(im, dm, sm)
    if max_match == sm and cost == 0:
        best_action = 'equal'
        lowest_cost = sm
    elif max_match == sm and cost == 1:
        best_action = 'replace'
        lowest_cost = sm
    elif max_match == im and ic < lowest_cost:
        best_action = 'insert'
        lowest_cost = ic
    elif max_match == dm and dc < lowest_cost:
        best_action = 'delete'
        lowest_cost = dc
    return best_action

class SequenceMatcher:
    """Similar to the difflib SequenceMatcher, but uses Levenshtein/edit
    distance.

    Members:
      a - first sequence
      b - second sequence
      opcodes - the cached opcodes
      dist - the cached distance value
      _matches - the cached match count
    """

    def __init__(self, a=None, b=None, test=operator.eq,
                 action_function=lowest_cost_action):
        """Initialize the object with sequences a and b.  Optionally, one can
        specify a test function that is used to compare sequence elements.
        This defaults to the built in eq operator (i.e. operator.eq).
        """
        if a is None:
            a = []
        if b is None:
            b = []
        self.seq1 = a
        self.seq2 = b
        self._reset_object()
        self.action_function = action_function

    def set_seqs(self, a, b):
        """Specify two alternative sequences -- reset any cached values also."""
        self.set_seq1(a)
        self.set_seq2(b)
        self._reset_object()

    def _reset_object(self):
        """Clear out the cached values for distance, matches, and opcodes."""
        self.opcodes = None
        self.dist = None
        self._matches = None

    def set_seq1(self, a):
        """Specify a new sequence for sequence 1."""
        self._reset_object()
        self.seq1 = a

    def set_seq2(self, b):
        """Specify a new sequence for sequence 2."""
        self._reset_object()
        self.seq2 = b

    def find_longest_match(self, alo, ahi, blo, bhi):
        """Not implemented!"""
        raise NotImplementedError()

    def get_matching_blocks(self):
        """Similar to get_opcodes(), but returns only the opcodes that are
        'equal' and returns them in a somewhat different format
        (i.e. (i, j, n) )."""
        opcodes = self.get_opcodes()
        match_opcodes = filter(lambda x: x[0] == 'equal', opcodes)
        return map(lambda opcode: [opcode[1], opcode[3], opcode[2] - opcode[1]],
                   match_opcodes)

    def get_opcodes(self):
        """Returns a list of opcodes.  Opcodes are the same as defined by
        difflib."""
        if not self.opcodes:
            d, m, opcodes = edit_distance_backpointer(self.seq1, self.seq2,
                                                      action_function=self.action_function)
            if self.dist:
                assert (d == self.dist)
            if self._matches:
                assert (m == self._matches)
            self.dist = d
            self._matches = m
            self.opcodes = opcodes
        return self.opcodes

    def get_grouped_opcodes(self, n=None):
        """Not implemented!"""
        raise NotImplementedError()

    def ratio(self):
        """Ratio of matches to the average sequence length"""
        return 2.0 * self.matches() / (len(self.seq1) + len(self.seq2))

    def quick_ratio(self):
        """Same as ratio()"""
        return self.ratio()

    def real_quick_ratio(self):
        """Same as ratio()"""
        return self.ratio()

    def _compute_distance_fast(self):
        """Calls edit_distance, and asserts that if we already have values for
        matches and distance, that they match."""
        d, m = edit_distance(self.seq1, self.seq2,
                             action_function=self.action_function)
        if self.dist:
            assert (d == self.dist)
        if self._matches:
            assert (m == self._matches)
        self.dist = d
        self._matches = m

    def distance(self):
        """Returns the edit distance of the two loaded sequences.  This should
        be a little faster than getting the same information from
        get_opcodes()."""
        if not self.dist:
            self._compute_distance_fast()
        return self.dist

    def matches(self):
        """Returns the number of matches in the alignment of the two sequences.
        This should be a little faster than getting the same information from
        get_opcodes()"""
        if not self._matches:
            self._compute_distance_fast()
        return self._matches

def edit_distance(seq1, seq2, action_function=lowest_cost_action):
    """Computes the edit distance between the two given sequences.
    This uses the relatively 'fast' method that only constructs
    two columns of the 2d array.  This function actually uses four columns
    because we track the number of matches too.
    """
    m = len(seq1)
    n = len(seq2)
    # Special, easy cases:
    if seq1 == seq2:
        return 0, n
    if m == 0:
        return n, 0
    if n == 0:
        return m, 0
    v0 = [0] * (n + 1)     # The two 'error' columns
    v1 = [0] * (n + 1)
    m0 = [0] * (n + 1)     # The two 'match' columns
    m1 = [0] * (n + 1)
    for i in range(1, n + 1):
        v0[i] = i
    for i in range(1, m + 1):
        v1[0] = i + 1
        for j in range(1, n + 1):
            cost = 0 if seq1[i - 1] == seq2[j - 1] else 1
            # The costs
            ins_cost = v1[j - 1] + 1
            del_cost = v0[j] + 1
            sub_cost = v0[j - 1] + cost
            # Match counts
            ins_match = m1[j - 1]
            del_match = m0[j]
            sub_match = m0[j - 1] + int(not cost)

            action = action_function(ins_cost, del_cost, sub_cost, ins_match,
                                     del_match, sub_match, cost)

            if action in ['equal', 'replace']:
                v1[j] = sub_cost
                m1[j] = sub_match
            elif action == 'insert':
                v1[j] = ins_cost
                m1[j] = ins_match
            elif action == 'delete':
                v1[j] = del_cost
                m1[j] = del_match
            else:
                raise Exception('Invalid dynamic programming option returned!')
                # Copy the columns over
        for i in range(0, n + 1):
            v0[i] = v1[i]
            m0[i] = m1[i]
    return v1[n], m1[n]

def edit_distance_backpointer(seq1, seq2, action_function=lowest_cost_action):
    """Similar to edit_distance() except that this function keeps backpointers
    during the search.  This allows us to return the opcodes (i.e. the specific
    edits that were used to change from one string to another.  This funtion
    contrusts the full 2d array (actually it contructs three of them... one
    for distances, one for matches, and one for backpointers."""
    matches = 0
    # Create a 2d distance array
    m = len(seq1)
    n = len(seq2)
    # distances array:
    d = [[0 for x in range(n + 1)] for y in range(m + 1)]
    # backpointer array:
    bp = [[None for x in range(n + 1)] for y in range(m + 1)]
    # matches array:
    matches = [[0 for x in range(n + 1)] for y in range(m + 1)]
    # source prefixes can be transformed into empty string by
    # dropping all characters
    for i in range(1, m + 1):
        d[i][0] = i
        bp[i][0] = ['delete', i - 1, i, 0, 0]
    # target prefixes can be reached from empty source prefix by inserting
    # every characters
    for j in range(1, n + 1):
        d[0][j] = j
        bp[0][j] = ['insert', 0, 0, j - 1, j]
    # compute the edit distance...
    for i in range(1, m + 1):
        for j in range(1, n + 1):

            cost = 0 if seq1[i - 1] == seq2[j - 1] else 1
            # The costs of each action...
            ins_cost = d[i][j - 1] + 1       # insertion
            del_cost = d[i - 1][j] + 1       # deletion
            sub_cost = d[i - 1][j - 1] + cost  # substitution/match

            # The match scores of each action
            ins_match = matches[i][j - 1]
            del_match = matches[i - 1][j]
            sub_match = matches[i - 1][j - 1] + int(not cost)

            action = action_function(ins_cost, del_cost, sub_cost, ins_match,
                                     del_match, sub_match, cost)
            if action == 'equal':
                d[i][j] = sub_cost
                matches[i][j] = sub_match
                bp[i][j] = ['equal', i - 1, i, j - 1, j]
            elif action == 'replace':
                d[i][j] = sub_cost
                matches[i][j] = sub_match
                bp[i][j] = ['replace', i - 1, i, j - 1, j]
            elif action == 'insert':
                d[i][j] = ins_cost
                matches[i][j] = ins_match
                bp[i][j] = ['insert', i - 1, i - 1, j - 1, j]
            elif action == 'delete':
                d[i][j] = del_cost
                matches[i][j] = del_match
                bp[i][j] = ['delete', i - 1, i, j - 1, j - 1]
            else:
                raise Exception('Invalid dynamic programming action returned!')

    opcodes = get_opcodes_from_bp_table(bp)
    return d[m][n], matches[m][n], opcodes

def get_opcodes_from_bp_table(bp):
    """Given a 2d list structure, collect the opcodes from the best path."""
    x = len(bp) - 1
    y = len(bp[0]) - 1
    opcodes = []
    while x != 0 or y != 0:
        this_bp = bp[x][y]
        opcodes.append(this_bp)
        if this_bp[0] == 'equal' or this_bp[0] == 'replace':
            x = x - 1
            y = y - 1
        elif this_bp[0] == 'insert':
            y = y - 1
        elif this_bp[0] == 'delete':
            x = x - 1
    opcodes.reverse()
    return opcodes

def main():
    """Read two files line-by-line and print edit distances between each pair
    of lines. Will terminate at the end of the shorter of the two files."""

    file1 = sys.argv[1]
    file2 = sys.argv[2]

    print(file1)
    print(file2)

    with open(file1) as f1, open(file2) as f2:
        for line1, line2 in zip(f1, f2):
            print(line1.strip())
            print(line2.strip())
            print(edit_distance_backpointer(line1.split(), line2.split()))
            print(edit_distance(line1.split(), line2.split()))

if __name__ == "__main__":
    main()
<commit_msg>Fixed usage of a custom equality function.<commit_after># -*- mode: Python;-*-

import sys
import operator

# Cost is basically: was there a match or not.
# The other numbers are cumulative costs and matches...

def lowest_cost_action(ic, dc, sc, im, dm, sm, cost):
    """Given the following values, choose the action (insertion, deletion,
    or substitution), that results in the lowest cost (ties are broken using
    the 'match' score).  This is used within the dynamic programming algorithm.
    * ic - insertion cost
    * dc - deletion cost
    * sc - substitution cost
    * im - insertion match (score)
    * dm - deletion match (score)
    * sm - substitution match (score)
    """
    best_action = None
    best_match_count = -1
    min_cost = min(ic, dc, sc)
    if min_cost == sc and cost == 0:
        best_action = 'equal'
        best_match_count = sm
    elif min_cost == sc and cost == 1:
        best_action = 'replace'
        best_match_count = sm
    elif min_cost == ic and im > best_match_count:
        best_action = 'insert'
        best_match_count = im
    elif min_cost == dc and dm > best_match_count:
        best_action = 'delete'
        best_match_count = dm
    return best_action

def highest_match_action(ic, dc, sc, im, dm, sm, cost):
    """Given the following values, choose the action (insertion, deletion, or
    substitution), that results in the highest match score (ties are broken
    using the distance values).  This is used within the dynamic programming
    algorithm.
    * ic - insertion cost
    * dc - deletion cost
    * sc - substitution cost
    * im - insertion match (score)
    * dm - deletion match (score)
    * sm - substitution match (score)
    """
    best_action = None
    lowest_cost = float("inf")
    max_match = max(im, dm, sm)
    if max_match == sm and cost == 0:
        best_action = 'equal'
        lowest_cost = sm
    elif max_match == sm and cost == 1:
        best_action = 'replace'
        lowest_cost = sm
    elif max_match == im and ic < lowest_cost:
        best_action = 'insert'
        lowest_cost = ic
    elif max_match == dm and dc < lowest_cost:
        best_action = 'delete'
        lowest_cost = dc
    return best_action

class SequenceMatcher:
    """Similar to the difflib SequenceMatcher, but uses Levenshtein/edit
    distance.

    Members:
      a - first sequence
      b - second sequence
      opcodes - the cached opcodes
      dist - the cached distance value
      _matches - the cached match count
    """

    def __init__(self, a=None, b=None, test=operator.eq,
                 action_function=lowest_cost_action):
        """Initialize the object with sequences a and b.  Optionally, one can
        specify a test function that is used to compare sequence elements.
        This defaults to the built in eq operator (i.e. operator.eq).
        """
        if a is None:
            a = []
        if b is None:
            b = []
        self.seq1 = a
        self.seq2 = b
        self._reset_object()
        self.action_function = action_function
        self.test = test

    def set_seqs(self, a, b):
        """Specify two alternative sequences -- reset any cached values also."""
        self.set_seq1(a)
        self.set_seq2(b)
        self._reset_object()

    def _reset_object(self):
        """Clear out the cached values for distance, matches, and opcodes."""
        self.opcodes = None
        self.dist = None
        self._matches = None

    def set_seq1(self, a):
        """Specify a new sequence for sequence 1."""
        self._reset_object()
        self.seq1 = a

    def set_seq2(self, b):
        """Specify a new sequence for sequence 2."""
        self._reset_object()
        self.seq2 = b

    def find_longest_match(self, alo, ahi, blo, bhi):
        """Not implemented!"""
        raise NotImplementedError()

    def get_matching_blocks(self):
        """Similar to get_opcodes(), but returns only the opcodes that are
        'equal' and returns them in a somewhat different format
        (i.e. (i, j, n) )."""
        opcodes = self.get_opcodes()
        match_opcodes = filter(lambda x: x[0] == 'equal', opcodes)
        return map(lambda opcode: [opcode[1], opcode[3], opcode[2] - opcode[1]],
                   match_opcodes)

    def get_opcodes(self):
        """Returns a list of opcodes.  Opcodes are the same as defined by
        difflib."""
        if not self.opcodes:
            d, m, opcodes = edit_distance_backpointer(self.seq1, self.seq2,
                                                      action_function=self.action_function,
                                                      test=self.test)
            if self.dist:
                assert (d == self.dist)
            if self._matches:
                assert (m == self._matches)
            self.dist = d
            self._matches = m
            self.opcodes = opcodes
        return self.opcodes

    def get_grouped_opcodes(self, n=None):
        """Not implemented!"""
        raise NotImplementedError()

    def ratio(self):
        """Ratio of matches to the average sequence length"""
        return 2.0 * self.matches() / (len(self.seq1) + len(self.seq2))

    def quick_ratio(self):
        """Same as ratio()"""
        return self.ratio()

    def real_quick_ratio(self):
        """Same as ratio()"""
        return self.ratio()

    def _compute_distance_fast(self):
        """Calls edit_distance, and asserts that if we already have values for
        matches and distance, that they match."""
        d, m = edit_distance(self.seq1, self.seq2,
                             action_function=self.action_function,
                             test=self.test)
        if self.dist:
            assert (d == self.dist)
        if self._matches:
            assert (m == self._matches)
        self.dist = d
        self._matches = m

    def distance(self):
        """Returns the edit distance of the two loaded sequences.  This should
        be a little faster than getting the same information from
        get_opcodes()."""
        if not self.dist:
            self._compute_distance_fast()
        return self.dist

    def matches(self):
        """Returns the number of matches in the alignment of the two sequences.
        This should be a little faster than getting the same information from
        get_opcodes()"""
        if not self._matches:
            self._compute_distance_fast()
        return self._matches

def edit_distance(seq1, seq2, action_function=lowest_cost_action, test=operator.eq):
    """Computes the edit distance between the two given sequences.
    This uses the relatively 'fast' method that only constructs
    two columns of the 2d array.  This function actually uses four columns
    because we track the number of matches too.
    """
    m = len(seq1)
    n = len(seq2)
    # Special, easy cases:
    if seq1 == seq2:
        return 0, n
    if m == 0:
        return n, 0
    if n == 0:
        return m, 0
    v0 = [0] * (n + 1)     # The two 'error' columns
    v1 = [0] * (n + 1)
    m0 = [0] * (n + 1)     # The two 'match' columns
    m1 = [0] * (n + 1)
    for i in range(1, n + 1):
        v0[i] = i
    for i in range(1, m + 1):
        v1[0] = i + 1
        for j in range(1, n + 1):
            cost = 0 if test(seq1[i - 1], seq2[j - 1]) else 1
            # The costs
            ins_cost = v1[j - 1] + 1
            del_cost = v0[j] + 1
            sub_cost = v0[j - 1] + cost
            # Match counts
            ins_match = m1[j - 1]
            del_match = m0[j]
            sub_match = m0[j - 1] + int(not cost)

            action = action_function(ins_cost, del_cost, sub_cost, ins_match,
                                     del_match, sub_match, cost)

            if action in ['equal', 'replace']:
                v1[j] = sub_cost
                m1[j] = sub_match
            elif action == 'insert':
                v1[j] = ins_cost
                m1[j] = ins_match
            elif action == 'delete':
                v1[j] = del_cost
                m1[j] = del_match
            else:
                raise Exception('Invalid dynamic programming option returned!')
                # Copy the columns over
        for i in range(0, n + 1):
            v0[i] = v1[i]
            m0[i] = m1[i]
    return v1[n], m1[n]

def edit_distance_backpointer(seq1, seq2, action_function=lowest_cost_action, test=operator.eq):
    """Similar to edit_distance() except that this function keeps backpointers
    during the search.  This allows us to return the opcodes (i.e. the specific
    edits that were used to change from one string to another.  This funtion
    contrusts the full 2d array (actually it contructs three of them... one
    for distances, one for matches, and one for backpointers."""
    matches = 0
    # Create a 2d distance array
    m = len(seq1)
    n = len(seq2)
    # distances array:
    d = [[0 for x in range(n + 1)] for y in range(m + 1)]
    # backpointer array:
    bp = [[None for x in range(n + 1)] for y in range(m + 1)]
    # matches array:
    matches = [[0 for x in range(n + 1)] for y in range(m + 1)]
    # source prefixes can be transformed into empty string by
    # dropping all characters
    for i in range(1, m + 1):
        d[i][0] = i
        bp[i][0] = ['delete', i - 1, i, 0, 0]
    # target prefixes can be reached from empty source prefix by inserting
    # every characters
    for j in range(1, n + 1):
        d[0][j] = j
        bp[0][j] = ['insert', 0, 0, j - 1, j]
    # compute the edit distance...
    for i in range(1, m + 1):
        for j in range(1, n + 1):

            cost = 0 if test(seq1[i - 1], seq2[j - 1]) else 1
            # The costs of each action...
            ins_cost = d[i][j - 1] + 1       # insertion
            del_cost = d[i - 1][j] + 1       # deletion
            sub_cost = d[i - 1][j - 1] + cost  # substitution/match

            # The match scores of each action
            ins_match = matches[i][j - 1]
            del_match = matches[i - 1][j]
            sub_match = matches[i - 1][j - 1] + int(not cost)

            action = action_function(ins_cost, del_cost, sub_cost, ins_match,
                                     del_match, sub_match, cost)
            if action == 'equal':
                d[i][j] = sub_cost
                matches[i][j] = sub_match
                bp[i][j] = ['equal', i - 1, i, j - 1, j]
            elif action == 'replace':
                d[i][j] = sub_cost
                matches[i][j] = sub_match
                bp[i][j] = ['replace', i - 1, i, j - 1, j]
            elif action == 'insert':
                d[i][j] = ins_cost
                matches[i][j] = ins_match
                bp[i][j] = ['insert', i - 1, i - 1, j - 1, j]
            elif action == 'delete':
                d[i][j] = del_cost
                matches[i][j] = del_match
                bp[i][j] = ['delete', i - 1, i, j - 1, j - 1]
            else:
                raise Exception('Invalid dynamic programming action returned!')

    opcodes = get_opcodes_from_bp_table(bp)
    return d[m][n], matches[m][n], opcodes

def get_opcodes_from_bp_table(bp):
    """Given a 2d list structure, collect the opcodes from the best path."""
    x = len(bp) - 1
    y = len(bp[0]) - 1
    opcodes = []
    while x != 0 or y != 0:
        this_bp = bp[x][y]
        opcodes.append(this_bp)
        if this_bp[0] == 'equal' or this_bp[0] == 'replace':
            x = x - 1
            y = y - 1
        elif this_bp[0] == 'insert':
            y = y - 1
        elif this_bp[0] == 'delete':
            x = x - 1
    opcodes.reverse()
    return opcodes

def main():
    """Read two files line-by-line and print edit distances between each pair
    of lines. Will terminate at the end of the shorter of the two files."""

    file1 = sys.argv[1]
    file2 = sys.argv[2]

    print(file1)
    print(file2)

    with open(file1) as f1, open(file2) as f2:
        for line1, line2 in zip(f1, f2):
            print(line1.strip())
            print(line2.strip())
            print(edit_distance_backpointer(line1.split(), line2.split()))
            print(edit_distance(line1.split(), line2.split()))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
<Program Name>
  settings.py

<Author>
  Vladimir Diaz <vladimir.v.diaz@gmail.com>

<Started>
  December 7, 2016

<Copyright>
  See LICENSE for licensing information.

<Purpose>
  Store all crypto-related settings used by securesystemslib.
"""

# Help with Python 3 compatibility, where the print statement is a function, an
# implicit relative import is invalid, and the '/' operator performs true
# division.  Example:  print 'hello world' raises a 'SyntaxError' exception.
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

# Set a directory that should be used for all temporary files. If this
# is None, then the system default will be used. The system default
# will also be used if a directory path set here is invalid or
# unusable.
temporary_directory = None

# The current "good enough" number of PBKDF2 passphrase iterations.
# We recommend that important keys, such as root, be kept offline.
# 'toto.settings.PBKDF2_ITERATIONS' should increase as CPU speeds increase, set here
# at 100,000 iterations by default (in 2013).  The repository maintainer may opt
# to modify the default setting according to their security needs and
# computational restrictions.  A strong user password is still important.
# Modifying the number of iterations will result in a new derived key+PBDKF2
# combination if the key is loaded and re-saved, overriding any previous
# iteration setting used in the old '<keyid>' key file.
# https://en.wikipedia.org/wiki/PBKDF2
PBKDF2_ITERATIONS = 100000

# Supported cryptography libraries that can be used to generate and verify RSA
# keys and signatures:  ['pycrypto', 'pyca-cryptography']
RSA_CRYPTO_LIBRARY = 'pyca-cryptography'

# Supported Ed25519 cryptography libraries: ['pynacl', 'ed25519']
ED25519_CRYPTO_LIBRARY = 'ed25519'

# Supported ECDSA cryptography libraries: ['pyca-cryptography']
ECDSA_CRYPTO_LIBRARY = 'pyca-cryptography'

# General purpose cryptography. Algorithms and functions that fall under
# general purpose include AES, PBKDF2, cryptographically strong random number
# generators, and cryptographic hash functions.  The majority of the general
# cryptography is needed by the repository and developer tools.
# RSA_CRYPTO_LIBRARY and ED25519_CRYPTO_LIBRARY are needed on the client side
# of the software updater.
# Supported libraries for general-purpose cryptography:  ['pycrypto',
# 'pyca-cryptography']
GENERAL_CRYPTO_LIBRARY = 'pyca-cryptography'

# The algorithm(s) in HASH_ALGORITHMS are used to generate key IDs.
HASH_ALGORITHMS = ['sha256', 'sha512']
<commit_msg>Remove library configuration options from settings.py<commit_after>#!/usr/bin/env python

"""
<Program Name>
  settings.py

<Author>
  Vladimir Diaz <vladimir.v.diaz@gmail.com>

<Started>
  December 7, 2016

<Copyright>
  See LICENSE for licensing information.

<Purpose>
  Store all crypto-related settings used by securesystemslib.
"""

# Help with Python 3 compatibility, where the print statement is a function, an
# implicit relative import is invalid, and the '/' operator performs true
# division.  Example:  print 'hello world' raises a 'SyntaxError' exception.
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

# Set a directory that should be used for all temporary files. If this
# is None, then the system default will be used. The system default
# will also be used if a directory path set here is invalid or
# unusable.
temporary_directory = None

# The current "good enough" number of PBKDF2 passphrase iterations.
# We recommend that important keys, such as root, be kept offline.
# 'toto.settings.PBKDF2_ITERATIONS' should increase as CPU speeds increase, set here
# at 100,000 iterations by default (in 2013).  The repository maintainer may opt
# to modify the default setting according to their security needs and
# computational restrictions.  A strong user password is still important.
# Modifying the number of iterations will result in a new derived key+PBDKF2
# combination if the key is loaded and re-saved, overriding any previous
# iteration setting used in the old '<keyid>' key file.
# https://en.wikipedia.org/wiki/PBKDF2
PBKDF2_ITERATIONS = 100000

# The algorithm(s) in HASH_ALGORITHMS are used to generate key IDs.
HASH_ALGORITHMS = ['sha256', 'sha512']
<|endoftext|>