<commit_before>#!/usr/bin/env python
import asyncio
import datetime
import emoji
import os
import sys

from telethon import TelegramClient
from telethon.tl.types import MessageMediaDocument
from telethon.tl.types import MessageMediaPhoto
from telethon.tl.types import MessageMediaWebPage
from telethon.tl.types import MessageMediaGeo
from telethon.tl.types import MessageMediaUnsupported
from telethon.tl.types import MessageMediaContact
from telethon.tl.types import WebPageEmpty
from config import API_ID, API_HASH, PHONE_NUM, SESSION_ID, CHAT_ID

MEDIA_DIR = "media"
AUDIO_DIR = "audio"
VIDEO_DIR = "video"
IMG_DIR = "image"
WEB_DIR = "web"
MONTHS_DICT = {
    1: "Gener",
    2: "Febrer",
    3: "Març",
    4: "Abril",
    5: "Maig",
    6: "Juny",
    7: "Juliol",
    8: "Agost",
    9: "Setembre",
    10: "Octubre",
    11: "Novembre",
    12: "Desembre"
}

class Message(object):
    def __init__(self, message):
        self.message = message

def create_session(session_id, phone_num, api_id, api_hash):
    """Creates a Telegram session."""
    client = TelegramClient(session_id, api_id, api_hash)
    client.start()
    return client

def format_media_path(msg, name, new_dir, extension):
    """Formats the file containing the media."""
    new_path = "{}-{}-{} {}:{:02d}:{:02d}_{}.{}".format(
        msg.date.year, msg.date.month, msg.date.day,
        msg.date.hour, msg.date.minute, msg.date.second, name,
        extension)
    return os.path.join(new_dir, new_path)

def download_media(msg, name, client, media_dir=MEDIA_DIR, audio_dir=AUDIO_DIR,
                   video_dir=VIDEO_DIR, img_dir=IMG_DIR, web_dir=WEB_DIR):
    """Downloads media, if any."""
    os.makedirs(media_dir, exist_ok=True)
    audio_dir = os.path.join(media_dir, audio_dir)
    os.makedirs(audio_dir, exist_ok=True)
    img_dir = os.path.join(media_dir, img_dir)
    os.makedirs(img_dir, exist_ok=True)
    video_dir = os.path.join(media_dir, video_dir)
    os.makedirs(video_dir, exist_ok=True)

    out_msg = ""

    if isinstance(msg.media, MessageMediaWebPage):
        content = msg.message
        if not isinstance(msg.media.webpage, WebPageEmpty):
            content = "\\url{{{}}} ".format(msg.media.webpage.url)
        out_msg = get_message_string(msg, name, content)
    elif isinstance(msg.media, MessageMediaGeo):
        content = "Geolocation({}, {})".format(
            msg.media.geo.long,
            msg.media.geo.lat)
        out_msg = get_message_string(msg, name, content)
    elif isinstance(msg.media, MessageMediaPhoto):
        path = format_media_path(msg, name, img_dir, "jpg")
        wait_fun(client.download_media, message=msg,
                 file="{}.jpg".format(path))
        horizontal = msg.media.photo.sizes[0].w > msg.media.photo.sizes[0].h
        latex_size = 0.5 if horizontal else 0.35
        out_msg = "\myfigure{%f}{%s}{%s}" % (
            latex_size, path, get_message_string(msg, name, msg.message))
    elif isinstance(msg.media, MessageMediaDocument):
        mimetype = msg.media.document.mime_type
        if mimetype == "video/mp4" or mimetype == "video/3gpp":
            ext = mimetype.split("/")[1]
            path = format_media_path(msg, name, video_dir, ext)
            wait_fun(client.download_media, message=msg,
                     file="{}".format(path))
            content = "(video a {})".format(path)
            try:
                # Try to download a thumbnail of the video
                path_thumb = format_media_path(msg, name, video_dir, "jpg")
                wait_fun(client.download_media, message=msg,
                        thumb=-1, file=path_thumb)
                out_msg = "\myfigure{0.3}{%s}{%s}" % (
                    path_thumb, get_message_string(msg, name,
                                                content + msg.message))
            except TypeError:
                out_msg = get_message_string(msg, name, msg.message)
        elif mimetype == "audio/ogg" or mimetype == "audio/mpeg" or mimetype == "audio/amr":
            ext = mimetype.split("/")[1]
            ext = "mp3" if ext == "mpeg" else ext
            path = format_media_path(msg, name, audio_dir, ext)
            wait_fun(client.download_media, message=msg,
                     file="{}".format(path))
            content = "(\`audio a {})".format(path)
            out_msg = get_message_string(msg, name,
                                         content + msg.message)
        elif mimetype == "image/jpeg" or mimetype == "image/gif":
            # Not sure why some images are stored as such instead of MessageMediaPhoto
            ext = mimetype.split("/")[1]
            path = format_media_path(msg, name, img_dir, ext)
            wait_fun(client.download_media, message=msg,
                    file="{}.jpg".format(path))
            horizontal = msg.media.document.thumbs[0].w > msg.media.document.thumbs[0].h
            latex_size = 0.5 if horizontal else 0.35
            out_msg = "\myfigure{%f}{%s}{%s}" % (
                latex_size, path, get_message_string(msg, name, msg.message))
        elif mimetype == "application/pdf":
            # Do not download PDFs
            pass
        else:
            import ipdb; ipdb.set_trace()
            print("CACA DOCUMENT")
            print(msg)
            print(msg.media)
            sys.exit()
    elif isinstance(msg.media, MessageMediaContact):
        out_msg = get_message_string(msg, name, "(Contacte: {} {})".format(
            msg.media.first_name, msg.media.phone_number))
    elif isinstance(msg.media, MessageMediaUnsupported):
        out_msg = get_message_string(msg, name, "(Message not parsed) " + msg.message)
    else:
        import ipdb; ipdb.set_trace()
        print(msg)
        print(msg.media)
        sys.exit()

    return out_msg

def get_message_string(msg, name, content):
    """Gets the actual string, customly formatted."""
    if content is None:
        content = ""
    else:
        content = ": " + parse_emojis(content)
    return '[{}:{:02d}h] \\textbf{{{}}}{}'.format(
        msg.date.hour, msg.date.minute, name,
        content.replace("_", "\_"))

def parse_emojis(in_str):
    """Adds the DejaSans prefix for LaTeX parsing."""
    out_str = ""
    for c in in_str:
        if c in emoji.UNICODE_EMOJI:
            out_str += "{\DejaSans " + c + "}"
        else:
            out_str += c
    return out_str

def add_new_day(date):
    return "\\textbf{{{}/{}/{}}}\n\n".format(
        date.day, date.month, date.year)

def add_new_month(date):
    return "\mychapter{%s del %d}\n\n" % \
        (MONTHS_DICT[date.month], date.year)

def get_chat(client, chat_id):
    """Gets the chat with the given id and the open client."""
    chat = None
    for d in client.iter_dialogs():
        if d.entity.id == CHAT_ID:
            chat = d.entity
            break
    return chat

def wait_fun(fun, **args):
    """Run function asynchronously and waits for it to finish."""
    loop = asyncio.get_event_loop()
    loop.run_until_complete(fun(**args))

def get_name(msg, ps_dict):
    """Returns name from the given message."""
    # Remove middle name if any
    try:
        first_name = ps_dict[msg.from_id].first_name.split(" ")[0]
    except KeyError:
        import ipdb; ipdb.set_trace()
        pass

    # Get only initial of last name
    last_initial = ps_dict[msg.from_id].last_name[0]
    return "{} {}".format(first_name, last_initial)

def get_participants(client, chat):
    """Gets a dictionary of participants in the given chat."""
    participants = {}
    for p in client.iter_participants(chat):
        participants[p.id] = p
        if p.first_name == "Bernat":
            # Bernat switched phone numbers, so let's hack the old one in
            participants[2390325] = p
    return participants

def process():
    """Main process."""
    # Open client and chat
    client = create_session(SESSION_ID, PHONE_NUM, API_ID, API_HASH)
    chat = get_chat(client, CHAT_ID)

    if chat is None:
        print("Group {} not found!".format(CHAT_ID))
        sys.exit()

    # Get all participants
    ps = get_participants(client, chat)

    # Download Chat Pic
    wait_fun(client.download_profile_photo, entity=chat, file='media/chat_pic.jpg')

    date = datetime.datetime.today()
    date = date.replace(day=27)
    date = date.replace(month=11)
    date = date.replace(year=2014)

    prev_month = None
    prev_day = None

    # Parse each message, from oldest to newest
    for message in client.iter_messages(chat, offset_date=date, reverse=True):
        # Empty string
        out_str = ""

        # Start a chapter for every new month
        if prev_month is None or prev_month != date.month:
            out_str += add_new_month(message.date)
            prev_month = date.month

        # Start a subsection for every new day
        if prev_day is None or prev_day != date.day:
            out_str += add_new_day(message.date)
            prev_day = date.day

        name = get_name(message, ps)
        if message.media:
            out_str += download_media(message, name, client)
        else:
            out_str += get_message_string(message, name, message.message)

        # Write to latex
        print(out_str)
        with open("latex/content.tex", "a") as f:
            f.write(out_str + '\n\n')
    print("Done")

if __name__ == "__main__":
    process()
<commit_msg>Added aac audio type support<commit_after>#!/usr/bin/env python
import asyncio
import datetime
import emoji
import os
import sys

from telethon import TelegramClient
from telethon.tl.types import MessageMediaDocument
from telethon.tl.types import MessageMediaPhoto
from telethon.tl.types import MessageMediaWebPage
from telethon.tl.types import MessageMediaGeo
from telethon.tl.types import MessageMediaUnsupported
from telethon.tl.types import MessageMediaContact
from telethon.tl.types import WebPageEmpty
from config import API_ID, API_HASH, PHONE_NUM, SESSION_ID, CHAT_ID

MEDIA_DIR = "media"
AUDIO_DIR = "audio"
VIDEO_DIR = "video"
IMG_DIR = "image"
WEB_DIR = "web"
MONTHS_DICT = {
    1: "Gener",
    2: "Febrer",
    3: "Març",
    4: "Abril",
    5: "Maig",
    6: "Juny",
    7: "Juliol",
    8: "Agost",
    9: "Setembre",
    10: "Octubre",
    11: "Novembre",
    12: "Desembre"
}

class Message(object):
    def __init__(self, message):
        self.message = message

def create_session(session_id, phone_num, api_id, api_hash):
    """Creates a Telegram session."""
    client = TelegramClient(session_id, api_id, api_hash)
    client.start()
    return client

def format_media_path(msg, name, new_dir, extension):
    """Formats the file containing the media."""
    new_path = "{}-{}-{} {}:{:02d}:{:02d}_{}.{}".format(
        msg.date.year, msg.date.month, msg.date.day,
        msg.date.hour, msg.date.minute, msg.date.second, name,
        extension)
    return os.path.join(new_dir, new_path)

def download_media(msg, name, client, media_dir=MEDIA_DIR, audio_dir=AUDIO_DIR,
                   video_dir=VIDEO_DIR, img_dir=IMG_DIR, web_dir=WEB_DIR):
    """Downloads media, if any."""
    os.makedirs(media_dir, exist_ok=True)
    audio_dir = os.path.join(media_dir, audio_dir)
    os.makedirs(audio_dir, exist_ok=True)
    img_dir = os.path.join(media_dir, img_dir)
    os.makedirs(img_dir, exist_ok=True)
    video_dir = os.path.join(media_dir, video_dir)
    os.makedirs(video_dir, exist_ok=True)

    out_msg = ""

    if isinstance(msg.media, MessageMediaWebPage):
        content = msg.message
        if not isinstance(msg.media.webpage, WebPageEmpty):
            content = "\\url{{{}}} ".format(msg.media.webpage.url)
        out_msg = get_message_string(msg, name, content)
    elif isinstance(msg.media, MessageMediaGeo):
        content = "Geolocation({}, {})".format(
            msg.media.geo.long,
            msg.media.geo.lat)
        out_msg = get_message_string(msg, name, content)
    elif isinstance(msg.media, MessageMediaPhoto):
        path = format_media_path(msg, name, img_dir, "jpg")
        wait_fun(client.download_media, message=msg,
                 file="{}.jpg".format(path))
        horizontal = msg.media.photo.sizes[0].w > msg.media.photo.sizes[0].h
        latex_size = 0.5 if horizontal else 0.35
        out_msg = "\myfigure{%f}{%s}{%s}" % (
            latex_size, path, get_message_string(msg, name, msg.message))
    elif isinstance(msg.media, MessageMediaDocument):
        mimetype = msg.media.document.mime_type
        if mimetype == "video/mp4" or mimetype == "video/3gpp":
            ext = mimetype.split("/")[1]
            path = format_media_path(msg, name, video_dir, ext)
            wait_fun(client.download_media, message=msg,
                     file="{}".format(path))
            content = "(video a {})".format(path)
            try:
                # Try to download a thumbnail of the video
                path_thumb = format_media_path(msg, name, video_dir, "jpg")
                wait_fun(client.download_media, message=msg,
                        thumb=-1, file=path_thumb)
                out_msg = "\myfigure{0.3}{%s}{%s}" % (
                    path_thumb, get_message_string(msg, name,
                                                content + msg.message))
            except TypeError:
                out_msg = get_message_string(msg, name, msg.message)
        elif mimetype == "audio/ogg" or mimetype == "audio/mpeg" or \
                mimetype == "audio/amr" or mimetype == "audio/aac-adts":
            ext = mimetype.split("/")[1]
            ext = "mp3" if ext == "mpeg" else ext
            ext = "aac" if ext == "aac-adts" else ext
            path = format_media_path(msg, name, audio_dir, ext)
            wait_fun(client.download_media, message=msg,
                     file="{}".format(path))
            content = "(\`audio a {})".format(path)
            out_msg = get_message_string(msg, name,
                                         content + msg.message)
        elif mimetype == "image/jpeg" or mimetype == "image/gif":
            # Not sure why some images are stored as such instead of MessageMediaPhoto
            ext = mimetype.split("/")[1]
            path = format_media_path(msg, name, img_dir, ext)
            wait_fun(client.download_media, message=msg,
                    file="{}.jpg".format(path))
            horizontal = msg.media.document.thumbs[0].w > msg.media.document.thumbs[0].h
            latex_size = 0.5 if horizontal else 0.35
            out_msg = "\myfigure{%f}{%s}{%s}" % (
                latex_size, path, get_message_string(msg, name, msg.message))
        elif mimetype == "application/pdf":
            # Do not download PDFs
            pass
        else:
            import ipdb; ipdb.set_trace()
            print("CACA DOCUMENT")
            print(msg)
            print(msg.media)
            sys.exit()
    elif isinstance(msg.media, MessageMediaContact):
        out_msg = get_message_string(msg, name, "(Contacte: {} {})".format(
            msg.media.first_name, msg.media.phone_number))
    elif isinstance(msg.media, MessageMediaUnsupported):
        out_msg = get_message_string(msg, name, "(Message not parsed) " + msg.message)
    else:
        import ipdb; ipdb.set_trace()
        print(msg)
        print(msg.media)
        sys.exit()

    return out_msg

def get_message_string(msg, name, content):
    """Gets the actual string, customly formatted."""
    if content is None:
        content = ""
    else:
        content = ": " + parse_emojis(content)
    return '[{}:{:02d}h] \\textbf{{{}}}{}'.format(
        msg.date.hour, msg.date.minute, name,
        content.replace("_", "\_"))

def parse_emojis(in_str):
    """Adds the DejaSans prefix for LaTeX parsing."""
    out_str = ""
    for c in in_str:
        if c in emoji.UNICODE_EMOJI:
            out_str += "{\DejaSans " + c + "}"
        else:
            out_str += c
    return out_str

def add_new_day(date):
    return "\\textbf{{{}/{}/{}}}\n\n".format(
        date.day, date.month, date.year)

def add_new_month(date):
    return "\mychapter{%s del %d}\n\n" % \
        (MONTHS_DICT[date.month], date.year)

def get_chat(client, chat_id):
    """Gets the chat with the given id and the open client."""
    chat = None
    for d in client.iter_dialogs():
        if d.entity.id == CHAT_ID:
            chat = d.entity
            break
    return chat

def wait_fun(fun, **args):
    """Run function asynchronously and waits for it to finish."""
    loop = asyncio.get_event_loop()
    loop.run_until_complete(fun(**args))

def get_name(msg, ps_dict):
    """Returns name from the given message."""
    # Remove middle name if any
    try:
        first_name = ps_dict[msg.from_id].first_name.split(" ")[0]
    except KeyError:
        import ipdb; ipdb.set_trace()
        pass

    # Get only initial of last name
    last_initial = ps_dict[msg.from_id].last_name[0]
    return "{} {}".format(first_name, last_initial)

def get_participants(client, chat):
    """Gets a dictionary of participants in the given chat."""
    participants = {}
    for p in client.iter_participants(chat):
        participants[p.id] = p
        if p.first_name == "Bernat":
            # Bernat switched phone numbers, so let's hack the old one in
            participants[2390325] = p
    return participants

def process():
    """Main process."""
    # Open client and chat
    client = create_session(SESSION_ID, PHONE_NUM, API_ID, API_HASH)
    chat = get_chat(client, CHAT_ID)

    if chat is None:
        print("Group {} not found!".format(CHAT_ID))
        sys.exit()

    # Get all participants
    ps = get_participants(client, chat)

    # Download Chat Pic
    wait_fun(client.download_profile_photo, entity=chat, file='media/chat_pic.jpg')

    date = datetime.datetime.today()
    date = date.replace(day=26)
    date = date.replace(month=12)
    date = date.replace(year=2014)

    prev_month = None
    prev_day = None

    # Parse each message, from oldest to newest
    for message in client.iter_messages(chat, offset_date=date, reverse=True):
        # Empty string
        out_str = ""

        # Start a chapter for every new month
        if prev_month is None or prev_month != date.month:
            out_str += add_new_month(message.date)
            prev_month = date.month

        # Start a subsection for every new day
        if prev_day is None or prev_day != date.day:
            out_str += add_new_day(message.date)
            prev_day = date.day

        name = get_name(message, ps)
        if message.media:
            out_str += download_media(message, name, client)
        else:
            out_str += get_message_string(message, name, message.message)

        # Write to latex
        print(out_str)
        with open("latex/content.tex", "a") as f:
            f.write(out_str + '\n\n')
    print("Done")

if __name__ == "__main__":
    process()
<|endoftext|>


<commit_before>import sys
import os
import re

## store data
import json
import pickle

urlpat = r'((http[s]?):\/\/)?(\w+\.)*(?P<domain>\w+)\.(\w+)(\/.*)?'

error = open( 'error.log', 'w' )

def download( id, url, storeto ):

    url = url.strip()

    try:
        ## try to dynamically load the correct script using the domain name
        loader = re.match( urlpat , url ).group('domain')

        loader = __import__( 'sites.' + loader, fromlist = [ loader ] )

        story = loader.parse( url )

        if 'json' in storeto:
            json.dump( story , open( 'data/' + str(id) + '.json', 'w' ) )

        if 'pickle' in storeto:
            pickle.dump( story , open( 'data/' + str(id) + '.pickle', 'w' ) )

        return story['http']

    except Exception, e:
        print e
        print "Failed " + url

        error.write( url + '\n' )

if __name__ == '__main__':

    import collections

    http_status = collections.defaultdict( int )

    for f in sys.argv[1:]:

        f = open( f )
        for id, url in enumerate( f ):

            s = download( id, url, 'pickle' )

            http_status[ s ] += 1

    print 'Final status'
    for s, c in http_status.items():
        print s, '\t', c
<commit_msg>Automatically create data folder<commit_after>import sys
import os
import re

## store data
import json
import pickle

urlpat = r'((http[s]?):\/\/)?(\w+\.)*(?P<domain>\w+)\.(\w+)(\/.*)?'

error = open( 'error.log', 'w' )

data_dir = 'data/'

if not os.path.exists( data_dir ):
    os.makedirs( data_dir )

def download( id, url, storeto ):

    url = url.strip()

    try:
        ## try to dynamically load the correct script using the domain name
        loader = re.match( urlpat , url ).group('domain')

        loader = __import__( 'sites.' + loader, fromlist = [ loader ] )

        story = loader.parse( url )

        if 'json' in storeto:
            json.dump( story , open( data_dir + str(id) + '.json', 'w' ) )

        if 'pickle' in storeto:
            pickle.dump( story , open( data_dir + str(id) + '.pickle', 'w' ) )

        return story['http']

    except Exception, e:
        print e
        print "Failed " + url

        error.write( url + '\n' )

if __name__ == '__main__':

    import collections

    http_status = collections.defaultdict( int )

    for f in sys.argv[1:]:

        f = open( f )
        for id, url in enumerate( f ):

            s = download( id, url, 'pickle' )

            http_status[ s ] += 1

    print 'Final status'
    for s, c in http_status.items():
        print s, '\t', c
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os

from telemetry.core import util
from telemetry.page import page as page_module
from telemetry.page.actions import scroll
from telemetry.unittest import tab_test_case

class ScrollActionTest(tab_test_case.TabTestCase):
  def setUp(self):
    self._extra_browser_args.append('--enable-gpu-benchmarking')
    super(ScrollActionTest, self).setUp()

  def CreateAndNavigateToPageFromUnittestDataDir(
    self, filename, page_attributes):
    self._browser.SetHTTPServerDirectories(util.GetUnittestDataDir())
    page = page_module.Page(
      self._browser.http_server.UrlOf(filename),
      None, # In this test, we don't need a page set.
      attributes=page_attributes)

    self._tab.Navigate(page.url)
    self._tab.WaitForDocumentReadyStateToBeComplete()

    return page

  def testScrollAction(self):
    page = self.CreateAndNavigateToPageFromUnittestDataDir(
        "blank.html",
        page_attributes={"smoothness": {
          "action": "scroll"
          }})
    # Make page bigger than window so it's scrollable.
    self._tab.ExecuteJavaScript("""document.body.style.height =
                              (2 * window.innerHeight + 1) + 'px';""")

    self.assertEquals(
        self._tab.EvaluateJavaScript("""document.documentElement.scrollTop
                                   || document.body.scrollTop"""), 0)

    i = scroll.ScrollAction()
    i.WillRunAction(page, self._tab)

    self._tab.ExecuteJavaScript("""
        window.__scrollAction.beginMeasuringHook = function() {
            window.__didBeginMeasuring = true;
        };
        window.__scrollAction.endMeasuringHook = function() {
            window.__didEndMeasuring = true;
        };""")
    i.RunAction(page, self._tab, None)

    self.assertTrue(self._tab.EvaluateJavaScript('window.__didBeginMeasuring'))
    self.assertTrue(self._tab.EvaluateJavaScript('window.__didEndMeasuring'))

    # Allow for roundoff error in scaled viewport.
    scroll_position = self._tab.EvaluateJavaScript(
        """(document.documentElement.scrollTop || document.body.scrollTop)
        + window.innerHeight""")
    scroll_height = self._tab.EvaluateJavaScript('document.body.scrollHeight')
    difference = scroll_position - scroll_height
    self.assertTrue(abs(difference) <= 1)

  def testBoundingClientRect(self):
    self.CreateAndNavigateToPageFromUnittestDataDir('blank.html', {})
    with open(os.path.join(os.path.dirname(__file__), 'scroll.js')) as f:
      js = f.read()
      self._tab.ExecuteJavaScript(js)

    # Verify that the rect returned by getBoundingVisibleRect() in scroll.js is
    # completely contained within the viewport. Scroll events dispatched by the
    # scrolling API use the center of this rect as their location, and this
    # location needs to be within the viewport bounds to correctly decide
    # between main-thread and impl-thread scroll. If the scrollable area were
    # not clipped to the viewport bounds, then the instance used here (the
    # scrollable area being more than twice as tall as the viewport) would
    # result in a scroll location outside of the viewport bounds.
    self._tab.ExecuteJavaScript("""document.body.style.height =
                           (2 * window.innerHeight + 1) + 'px';""")

    rect_top = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).top'))
    rect_height = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).height'))
    rect_bottom = rect_top + rect_height

    rect_left = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).left'))
    rect_width = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).width'))
    rect_right = rect_left + rect_width

    viewport_height = int(self._tab.EvaluateJavaScript('window.innerHeight'))
    viewport_width = int(self._tab.EvaluateJavaScript('window.innerWidth'))

    self.assertTrue(rect_bottom <= viewport_height,
        msg='%s + %s <= %s' % (rect_top, rect_height, viewport_height))
    self.assertTrue(rect_right <= viewport_width,
        msg='%s + %s <= %s' % (rect_left, rect_width, viewport_width))
<commit_msg>Telemetry testScrollAction fails after a Blink roll 159695:159835<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os

from telemetry.core import util
from telemetry.page import page as page_module
from telemetry.page.actions import scroll
from telemetry.unittest import tab_test_case

class ScrollActionTest(tab_test_case.TabTestCase):
  def setUp(self):
    self._extra_browser_args.append('--enable-gpu-benchmarking')
    super(ScrollActionTest, self).setUp()

  def CreateAndNavigateToPageFromUnittestDataDir(
    self, filename, page_attributes):
    self._browser.SetHTTPServerDirectories(util.GetUnittestDataDir())
    page = page_module.Page(
      self._browser.http_server.UrlOf(filename),
      None, # In this test, we don't need a page set.
      attributes=page_attributes)

    self._tab.Navigate(page.url)
    self._tab.WaitForDocumentReadyStateToBeComplete()

    return page

#  crbug.com/308846, Blink roll 159695:159835.
#
#  def testScrollAction(self):
#    page = self.CreateAndNavigateToPageFromUnittestDataDir(
#        "blank.html",
#        page_attributes={"smoothness": {
#          "action": "scroll"
#          }})
#    # Make page bigger than window so it's scrollable.
#    self._tab.ExecuteJavaScript("""document.body.style.height =
#                              (2 * window.innerHeight + 1) + 'px';""")
#
#    self.assertEquals(
#        self._tab.EvaluateJavaScript("""document.documentElement.scrollTop
#                                   || document.body.scrollTop"""), 0)
#
#    i = scroll.ScrollAction()
#    i.WillRunAction(page, self._tab)
#
#    self._tab.ExecuteJavaScript("""
#        window.__scrollAction.beginMeasuringHook = function() {
#            window.__didBeginMeasuring = true;
#        };
#        window.__scrollAction.endMeasuringHook = function() {
#            window.__didEndMeasuring = true;
#        };""")
#    i.RunAction(page, self._tab, None)
#
#    self.assertTrue(self._tab.EvaluateJavaScript('window.__didBeginMeasuring'))
#    self.assertTrue(self._tab.EvaluateJavaScript('window.__didEndMeasuring'))
#
#    # Allow for roundoff error in scaled viewport.
#    scroll_position = self._tab.EvaluateJavaScript(
#        """(document.documentElement.scrollTop || document.body.scrollTop)
#        + window.innerHeight""")
#    scroll_height = self._tab.EvaluateJavaScript('document.body.scrollHeight')
#    difference = scroll_position - scroll_height
#    self.assertTrue(abs(difference) <= 1)

  def testBoundingClientRect(self):
    self.CreateAndNavigateToPageFromUnittestDataDir('blank.html', {})
    with open(os.path.join(os.path.dirname(__file__), 'scroll.js')) as f:
      js = f.read()
      self._tab.ExecuteJavaScript(js)

    # Verify that the rect returned by getBoundingVisibleRect() in scroll.js is
    # completely contained within the viewport. Scroll events dispatched by the
    # scrolling API use the center of this rect as their location, and this
    # location needs to be within the viewport bounds to correctly decide
    # between main-thread and impl-thread scroll. If the scrollable area were
    # not clipped to the viewport bounds, then the instance used here (the
    # scrollable area being more than twice as tall as the viewport) would
    # result in a scroll location outside of the viewport bounds.
    self._tab.ExecuteJavaScript("""document.body.style.height =
                           (2 * window.innerHeight + 1) + 'px';""")

    rect_top = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).top'))
    rect_height = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).height'))
    rect_bottom = rect_top + rect_height

    rect_left = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).left'))
    rect_width = int(self._tab.EvaluateJavaScript(
        '__ScrollAction_GetBoundingVisibleRect(document.body).width'))
    rect_right = rect_left + rect_width

    viewport_height = int(self._tab.EvaluateJavaScript('window.innerHeight'))
    viewport_width = int(self._tab.EvaluateJavaScript('window.innerWidth'))

    self.assertTrue(rect_bottom <= viewport_height,
        msg='%s + %s <= %s' % (rect_top, rect_height, viewport_height))
    self.assertTrue(rect_right <= viewport_width,
        msg='%s + %s <= %s' % (rect_left, rect_width, viewport_width))
<|endoftext|>


<commit_before>"""
The logic and workings behind the stow and unstow sub commands
"""

from collections import defaultdict
import pathlib
import dploy.actions as actions
import dploy.utils as utils
import dploy.exceptions as exceptions

class AbstractBaseSubCommand():
    """
    An abstract class to unify shared functionality in stow commands
    """

    # pylint: disable=too-many-arguments
    def __init__(self, subcmd, sources, dest, is_silent, is_dry_run):
        self.subcmd = subcmd
        self.actions = []
        self.execptions = []
        self.is_silent = is_silent
        self.is_dry_run = is_dry_run

        for source in sources:
            source_input = pathlib.Path(source)
            dest_input = pathlib.Path(dest)
            source_absolute = utils.get_absolute_path(source_input)
            dest_absolute = utils.get_absolute_path(dest_input)
            self.validate_input(source_input, dest_input)
            self.collect_actions(source_absolute, dest_absolute)

        self.check_for_conflicting_actions()
        self.execute_actions()

    def check_for_conflicting_actions(self):
        """
        todo
        """
        pass

    def validate_input(self, source, dest):
        """
        todo
        """
        pass

    def collect_actions(self, source, dest):
        """
        todo
        """
        pass

    def execute_actions(self):
        """
        todo
        """
        if len(self.execptions) > 0:
            for execption in self.execptions:
                raise execption
        else:
            for action in self.actions:
                if not self.is_silent:
                    print(action)
                if not self.is_dry_run:
                    action.execute()

class AbstractBaseStow(AbstractBaseSubCommand):
    """
    todo
    """
    # pylint: disable=too-many-arguments
    def __init__(self, subcmd, source, dest, is_silent, is_dry_run):
        self.is_unfolding = False
        super().__init__(subcmd, source, dest, is_silent, is_dry_run)

    def validate_input(self, source, dest):
        """
        todo
        """
        if not source.is_dir():
            raise exceptions.no_such_directory(self.subcmd, source)

        elif not dest.is_dir():
            raise exceptions.no_such_directory_to_subcmd_into(self.subcmd, dest)

        elif not utils.is_directory_readable(source):
            raise exceptions.insufficient_permissions_to_subcmd_from(self.subcmd, source)

        elif not utils.is_directory_writable(dest):
            raise exceptions.insufficient_permissions_to_subcmd_to(self.subcmd, dest)

    def get_directory_contents(self, directory):
        """
        get the contents of a directory while handling permission errors that
        may occur
        """
        contents = []

        try:
            contents = utils.get_directory_contents(directory)
        except PermissionError:
            self.execptions.append(exceptions.permission_denied(self.subcmd, directory))

        return contents

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        pass

    def are_directories(self, source, dest):
        """
        what to do if the source and dest are directories
        """
        pass

    def are_other(self, source, dest):
        """
        what to do if no particular condition is true cases are found
        """
        pass

    def collect_actions(self, source, dest):
        """
        todo
        """
        sources = self.get_directory_contents(source)

        for source in sources:
            dest_path = dest / pathlib.Path(source.name)

            if dest_path.exists():
                if utils.is_same_file(dest_path, source):
                    if dest_path.is_symlink() or self.is_unfolding:
                        self.are_same_file(source, dest_path)
                    else:
                        self.execptions.append(
                            exceptions.source_is_same_as_dest(self.subcmd, dest_path))

                elif dest_path.is_dir() and source.is_dir:
                    self.are_directories(source, dest_path)
                else:
                    self.execptions.append(
                        exceptions.conflicts_with_existing_file(self.subcmd, dest_path))

            elif dest_path.is_symlink():
                self.execptions.append(
                    exceptions.conflicts_with_existing_link(self.subcmd, dest_path))

            elif not dest_path.parent.exists() and not self.is_unfolding:
                self.execptions.append(exceptions.no_such_directory(self.subcmd, dest_path.parent))

            else:
                self.are_other(source, dest_path)

class UnStow(AbstractBaseStow):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("unstow", source, dest, is_silent, is_dry_run)

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        self.actions.append(actions.UnLink(self.subcmd, dest))

    def are_directories(self, source, dest):
        if not dest.is_symlink():
            self.collect_actions(source, dest)

    def are_other(self, source, dest):
        self.actions.append(actions.AlreadyUnlinked(self.subcmd, source, dest))

class Link(AbstractBaseSubCommand):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("link", [source], dest, is_silent, is_dry_run)

    def validate_input(self, source, dest):
        """
        todo
        """
        if not source.exists():
            raise exceptions.no_such_file_or_directory(self.subcmd, source)

        elif not dest.parent.exists():
            raise exceptions.no_such_directory(self.subcmd, dest.parent)

        elif (not utils.is_file_readable(source)
              or not utils.is_directory_readable(source)):
            raise exceptions.insufficient_permissions(self.subcmd, source)

        elif (not utils.is_file_writable(dest.parent)
              or not utils.is_directory_writable(dest.parent)):
            raise  exceptions.insufficient_permissions_to_subcmd_to(self.subcmd, dest)

    def collect_actions(self, source, dest):
        """
        todo
        """

        if dest.exists():
            if utils.is_same_file(dest, source):
                self.actions.append(actions.AlreadyLinked(self.subcmd,
                                                          source,
                                                          dest))
            else:
                self.execptions.append(exceptions.conflicts_with_existing_file(self.subcmd, dest))

        elif dest.is_symlink():
            self.execptions.append(exceptions.conflicts_with_existing_link(self.subcmd, dest))

        elif not dest.parent.exists():
            self.execptions.append(
                exceptions.no_such_directory_to_subcmd_into(self.subcmd, dest.parent))

        else:
            self.actions.append(actions.SymbolicLink(self.subcmd, source, dest))

class Stow(AbstractBaseStow):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("stow", source, dest, is_silent, is_dry_run)

    def unfold(self, source, dest):
        """
        todo
        """
        self.is_unfolding = True
        self.actions.append(actions.UnLink(self.subcmd, dest))
        self.actions.append(actions.MakeDirectory(self.subcmd, dest))
        self.collect_actions(source, dest)
        self.is_unfolding = False

    def list_duplicates(self):
        """
        todo
        """
        tally = defaultdict(list)
        for i, item in enumerate(self.actions):
            if isinstance(item, actions.SymbolicLink):
                tally[item.dest].append(i)
        return ((key, locs) for key, locs in tally.items()
                if len(locs) > 1)

    def check_for_conflicting_actions(self):
        """
        check for symbolic link actions that would cause conflicting symbolic
        links to the same destination.
        """
        dupes = []
        for dup in self.list_duplicates():
            dupes.append(dup)

        if len(dupes) == 0:
            return

        for _, indicies in dupes:
            first_index = indicies[0]
            if self.actions[first_index].source.is_dir():
                self.unfold(self.actions[first_index].source,
                            self.actions[first_index].dest)
                for index in indicies[1:]:
                    self.is_unfolding = True
                    self.collect_actions(self.actions[index].source,
                                         self.actions[index].dest)
                    self.is_unfolding = False
            else:
                self.execptions.append(
                    exceptions.conflicts_with_another_source(
                        self.subcmd, self.actions[first_index].source))
                return

        for _, indicies in dupes:
            for index in reversed(indicies[1:]):
                del self.actions[index]

        self.check_for_conflicting_actions()

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        if self.is_unfolding:
            self.actions.append(
                actions.SymbolicLink(self.subcmd, source, dest))
        else:
            self.actions.append(
                actions.AlreadyLinked(self.subcmd, source, dest))

    def are_directories(self, source, dest):
        if dest.is_symlink():
            self.unfold(dest.resolve(), dest)
        self.collect_actions(source, dest)

    def are_other(self, source, dest):
        self.actions.append(
            actions.SymbolicLink(self.subcmd, source, dest))
<commit_msg>unstow: add check for links to be folded to a link<commit_after>"""
The logic and workings behind the stow and unstow sub commands
"""

from collections import defaultdict
from collections import Counter
import pathlib
import dploy.actions as actions
import dploy.utils as utils
import dploy.exceptions as exceptions

class AbstractBaseSubCommand():
    """
    An abstract class to unify shared functionality in stow commands
    """

    # pylint: disable=too-many-arguments
    def __init__(self, subcmd, sources, dest, is_silent, is_dry_run):
        self.subcmd = subcmd
        self.actions = []
        self.execptions = []
        self.is_silent = is_silent
        self.is_dry_run = is_dry_run

        for source in sources:
            source_input = pathlib.Path(source)
            dest_input = pathlib.Path(dest)
            source_absolute = utils.get_absolute_path(source_input)
            dest_absolute = utils.get_absolute_path(dest_input)
            self.validate_input(source_input, dest_input)
            self.collect_actions(source_absolute, dest_absolute)

        self.check_for_other_actions()
        self.execute_actions()

    def check_for_other_actions(self):
        """
        todo
        """
        pass

    def validate_input(self, source, dest):
        """
        todo
        """
        pass

    def collect_actions(self, source, dest):
        """
        todo
        """
        pass

    def execute_actions(self):
        """
        todo
        """
        if len(self.execptions) > 0:
            for execption in self.execptions:
                raise execption
        else:
            for action in self.actions:
                if not self.is_silent:
                    print(action)
                if not self.is_dry_run:
                    action.execute()

class AbstractBaseStow(AbstractBaseSubCommand):
    """
    todo
    """
    # pylint: disable=too-many-arguments
    def __init__(self, subcmd, source, dest, is_silent, is_dry_run):
        self.is_unfolding = False
        super().__init__(subcmd, source, dest, is_silent, is_dry_run)

    def validate_input(self, source, dest):
        """
        todo
        """
        if not source.is_dir():
            raise exceptions.no_such_directory(self.subcmd, source)

        elif not dest.is_dir():
            raise exceptions.no_such_directory_to_subcmd_into(self.subcmd, dest)

        elif not utils.is_directory_readable(source):
            raise exceptions.insufficient_permissions_to_subcmd_from(self.subcmd, source)

        elif not utils.is_directory_writable(dest):
            raise exceptions.insufficient_permissions_to_subcmd_to(self.subcmd, dest)

    def get_directory_contents(self, directory):
        """
        get the contents of a directory while handling permission errors that
        may occur
        """
        contents = []

        try:
            contents = utils.get_directory_contents(directory)
        except PermissionError:
            self.execptions.append(exceptions.permission_denied(self.subcmd, directory))

        return contents

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        pass

    def are_directories(self, source, dest):
        """
        what to do if the source and dest are directories
        """
        pass

    def are_other(self, source, dest):
        """
        what to do if no particular condition is true cases are found
        """
        pass

    def collect_actions(self, source, dest):
        """
        todo
        """
        sources = self.get_directory_contents(source)

        for source in sources:
            dest_path = dest / pathlib.Path(source.name)

            if dest_path.exists():
                if utils.is_same_file(dest_path, source):
                    if dest_path.is_symlink() or self.is_unfolding:
                        self.are_same_file(source, dest_path)
                    else:
                        self.execptions.append(
                            exceptions.source_is_same_as_dest(self.subcmd, dest_path))

                elif dest_path.is_dir() and source.is_dir:
                    self.are_directories(source, dest_path)
                else:
                    self.execptions.append(
                        exceptions.conflicts_with_existing_file(self.subcmd, dest_path))

            elif dest_path.is_symlink():
                self.execptions.append(
                    exceptions.conflicts_with_existing_link(self.subcmd, dest_path))

            elif not dest_path.parent.exists() and not self.is_unfolding:
                self.execptions.append(exceptions.no_such_directory(self.subcmd, dest_path.parent))

            else:
                self.are_other(source, dest_path)

class UnStow(AbstractBaseStow):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("unstow", source, dest, is_silent, is_dry_run)

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        self.actions.append(actions.UnLink(self.subcmd, dest))

    def are_directories(self, source, dest):
        if not dest.is_symlink():
            self.collect_actions(source, dest)

    def are_other(self, source, dest):
        self.actions.append(actions.AlreadyUnlinked(self.subcmd, source, dest))

    def check_for_other_actions(self):
        self.collect_folding_actions()

    def collect_folding_actions(self):
        """
        find canidates for folding i.e. when replacing a directories containing
        links with a single symlink to the directory itself would work instead
        """

        unlink_actions = (
            [a for a in self.actions if isinstance(a, actions.UnLink)])
        unlink_actions_targets = [a.target for a in unlink_actions]
        unlink_actions_targets_parents = [a.target.parent for a in unlink_actions]

        for parent in unlink_actions_targets_parents:
            items = utils.get_directory_contents(parent)
            other_links = []
            source_parent = None

            for item in items:
                if item in unlink_actions_targets:
                    pass
                elif item.is_symlink():
                    source_parent = item.resolve().parent
                    other_links.append(item.resolve().parent)

            other_links_counter = Counter(other_links)
            if len(other_links_counter.keys()) == 1:
                self.fold(source_parent, parent)

    def fold(self, source, dest):
        """
        add the required actions for folding
        """
        rmdir_actions_targets = (
            [a.target for a in self.actions if isinstance(a, actions.RemoveDirectory)])

        if  dest not in rmdir_actions_targets:
            self.actions.append(actions.RemoveDirectory(self.subcmd, dest))
            self.actions.append(actions.SymbolicLink(self.subcmd, source, dest))

class Link(AbstractBaseSubCommand):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("link", [source], dest, is_silent, is_dry_run)

    def validate_input(self, source, dest):
        """
        todo
        """
        if not source.exists():
            raise exceptions.no_such_file_or_directory(self.subcmd, source)

        elif not dest.parent.exists():
            raise exceptions.no_such_directory(self.subcmd, dest.parent)

        elif (not utils.is_file_readable(source)
              or not utils.is_directory_readable(source)):
            raise exceptions.insufficient_permissions(self.subcmd, source)

        elif (not utils.is_file_writable(dest.parent)
              or not utils.is_directory_writable(dest.parent)):
            raise  exceptions.insufficient_permissions_to_subcmd_to(self.subcmd, dest)

    def collect_actions(self, source, dest):
        """
        todo
        """

        if dest.exists():
            if utils.is_same_file(dest, source):
                self.actions.append(actions.AlreadyLinked(self.subcmd,
                                                          source,
                                                          dest))
            else:
                self.execptions.append(exceptions.conflicts_with_existing_file(self.subcmd, dest))

        elif dest.is_symlink():
            self.execptions.append(exceptions.conflicts_with_existing_link(self.subcmd, dest))

        elif not dest.parent.exists():
            self.execptions.append(
                exceptions.no_such_directory_to_subcmd_into(self.subcmd, dest.parent))

        else:
            self.actions.append(actions.SymbolicLink(self.subcmd, source, dest))

class Stow(AbstractBaseStow):
    """
    todo
    """
    def __init__(self, source, dest, is_silent=True, is_dry_run=False):
        super().__init__("stow", source, dest, is_silent, is_dry_run)

    def unfold(self, source, dest):
        """
        todo
        """
        self.is_unfolding = True
        self.actions.append(actions.UnLink(self.subcmd, dest))
        self.actions.append(actions.MakeDirectory(self.subcmd, dest))
        self.collect_actions(source, dest)
        self.is_unfolding = False

    def list_duplicates(self):
        """
        todo
        """
        tally = defaultdict(list)
        for i, item in enumerate(self.actions):
            if isinstance(item, actions.SymbolicLink):
                tally[item.dest].append(i)
        return ((key, locs) for key, locs in tally.items()
                if len(locs) > 1)

    def check_for_conflicting_actions(self):
        """
        check for symbolic link actions that would cause conflicting symbolic
        links to the same destination.
        """
        dupes = []
        for dup in self.list_duplicates():
            dupes.append(dup)

        if len(dupes) == 0:
            return

        for _, indicies in dupes:
            first_index = indicies[0]
            if self.actions[first_index].source.is_dir():
                self.unfold(self.actions[first_index].source,
                            self.actions[first_index].dest)
                for index in indicies[1:]:
                    self.is_unfolding = True
                    self.collect_actions(self.actions[index].source,
                                         self.actions[index].dest)
                    self.is_unfolding = False
            else:
                self.execptions.append(
                    exceptions.conflicts_with_another_source(
                        self.subcmd, self.actions[first_index].source))
                return

        for _, indicies in dupes:
            for index in reversed(indicies[1:]):
                del self.actions[index]

        self.check_for_conflicting_actions()

    def check_for_other_actions(self):
        self.check_for_conflicting_actions()

    def are_same_file(self, source, dest):
        """
        what to do if source and dest are the same files
        """
        if self.is_unfolding:
            self.actions.append(
                actions.SymbolicLink(self.subcmd, source, dest))
        else:
            self.actions.append(
                actions.AlreadyLinked(self.subcmd, source, dest))

    def are_directories(self, source, dest):
        if dest.is_symlink():
            self.unfold(dest.resolve(), dest)
        self.collect_actions(source, dest)

    def are_other(self, source, dest):
        self.actions.append(
            actions.SymbolicLink(self.subcmd, source, dest))
<|endoftext|>


<commit_before>"""*
"""
import subprocess as sp
import collections

import rest
import json

def product_checkout_useful_string():
    import pdb; pdb.set_trace()
    sp.call('git add ../server/')
    useful_string = sp.getoutput("git checkout |grep 'server/'")
    return useful_string

def parse_useful_string():
    """Return a changed defaultdict
    """
    useful_lines = product_checkout_useful_string().split(sep='\n')
    changed_files = collections.defaultdict(list)
    for line in useful_lines:
        if line == '':
            continue
        pairs = line.split('\t')
        changed_files[pairs[0]].append(pairs[1])
    return changed_files

def dispatch_action():
    changed_files = parse_useful_string()
    adds = changed_files.get('A')
    if adds:
        do_adds(adds)
    modify = changed_files.get('M')
    if modify:
        do_modify(modify)
    import pdb; pdb.set_trace()
    if not changed_files == {}:
        do_commit(changed_files)

def modify_image(modify_file):
    """*
    """
    file_name = modify_file[modify_file.rindex('/') + 1:]
    file_type = modify_file[modify_file.rindex('.') + 1:]
    file_bin = open('../server/images/' + file_name, 'rb')
    info = {'name': file_name, 'type': file_type, 'bin': file_bin}
    rest.modify_image(info)

def modify_json(modify_file):
    """*
    """
    import pdb; pdb.set_trace()
    file_name = modify_file[modify_file.rindex('/') + 1: -5]
    info = json.load(open('../%s' % modify_file))
    info['name'] = file_name
    rest.modify_product(info)

def do_modify(modify):
    """*
    """
    for modify_file in modify:
        import pdb; pdb.set_trace()
        if modify_file.find('images') != -1:
            modify_image(modify_file)
        elif modify_file.find('products') != -1:
            modify_json(modify_file)

def add_image(add_file):
    """*
    """
    file_name = add_file[add_file.rindex('/') + 1:]
    file_type = add_file[add_file.rindex('.') + 1:]
    file_bin = open('../server/images/' + file_name, 'rb')
    info = {'name': file_name, 'type': file_type, 'bin': file_bin}
    rest.add_image(info)

def add_json(add_file):
    """*
    """
    import pdb; pdb.set_trace()
    file_name = add_file[add_file.rindex('/') + 1: -5]
    info = json.load(open('../%s' % add_file))
    info['name'] = file_name
    rest.add_product(info)

def do_adds(adds):
    """*
    """
    for add_file in adds:
        import pdb; pdb.set_trace()
        if add_file.find('images') != -1:
            add_image(add_file)
        elif add_file.find('products') != -1:
            add_json(add_file)

def do_commit(files):
    """*
    """
    import pdb; pdb.set_trace()
    sp.call('git commit -m "**** %s"' % str(files))
    pass

if __name__ == '__main__':
    dispatch_action()
<commit_msg>Use iconv to solve unicode name<commit_after>"""*
"""
import subprocess as sp
import collections

import rest
import json

def product_checkout_useful_string():
    import pdb; pdb.set_trace()
    sp.call('git add ../server/')
    useful_string = sp.getoutput("git checkout |grep 'server/' |iconv -f UTF-8")
    return useful_string

def parse_useful_string():
    """Return a changed defaultdict
    """
    useful_lines = product_checkout_useful_string().split(sep='\n')
    changed_files = collections.defaultdict(list)
    for line in useful_lines:
        if line == '':
            continue
        pairs = line.split('\t')
        changed_files[pairs[0]].append(pairs[1])
    return changed_files

def dispatch_action():
    changed_files = parse_useful_string()
    adds = changed_files.get('A')
    if adds:
        do_adds(adds)
    modify = changed_files.get('M')
    if modify:
        do_modify(modify)
    import pdb; pdb.set_trace()
    if not changed_files == {}:
        do_commit(changed_files)

def modify_image(modify_file):
    """*
    """
    file_name = modify_file[modify_file.rindex('/') + 1:]
    file_type = modify_file[modify_file.rindex('.') + 1:]
    file_bin = open('../server/images/' + file_name, 'rb')
    info = {'name': file_name, 'type': file_type, 'bin': file_bin}
    rest.modify_image(info)

def modify_json(modify_file):
    """*
    """
    import pdb; pdb.set_trace()
    file_name = modify_file[modify_file.rindex('/') + 1: -5]
    info = json.load(open('../%s' % modify_file))
    info['name'] = file_name
    rest.modify_product(info)

def do_modify(modify):
    """*
    """
    for modify_file in modify:
        import pdb; pdb.set_trace()
        if modify_file.find('images') != -1:
            modify_image(modify_file)
        elif modify_file.find('products') != -1:
            modify_json(modify_file)

def add_image(add_file):
    """*
    """
    file_name = add_file[add_file.rindex('/') + 1:]
    file_type = add_file[add_file.rindex('.') + 1:]
    file_bin = open('../server/images/' + file_name, 'rb')
    info = {'name': file_name, 'type': file_type, 'bin': file_bin}
    rest.add_image(info)

def add_json(add_file):
    """*
    """
    import pdb; pdb.set_trace()
    file_name = add_file[add_file.rindex('/') + 1: -5]
    info = json.load(open('../%s' % add_file))
    info['name'] = file_name
    rest.add_product(info)

def do_adds(adds):
    """*
    """
    for add_file in adds:
        import pdb; pdb.set_trace()
        if add_file.find('images') != -1:
            add_image(add_file)
        elif add_file.find('products') != -1:
            add_json(add_file)

def do_commit(files):
    """*
    """
    import pdb; pdb.set_trace()
    sp.call('git commit -m "**** %s"' % str(files))
    pass

if __name__ == '__main__':
    dispatch_action()
<|endoftext|>


<commit_before>import os
from keras.callbacks import LearningRateScheduler
from keras.callbacks import ModelCheckpoint
from keras.callbacks import CSVLogger

from datasets import DataManager
from models.experimental_loss import MultiboxLoss
from models import SSD300
from keras.optimizers import SGD
from utils.generator import ImageGenerator
from utils.boxes import create_prior_boxes
from utils.boxes import to_point_form
from utils.training_utils import LearningRateManager

# hyper-parameters
batch_size = 5
num_epochs = 250
step_values = 160, 190, 220
image_shape = (300, 300, 3)
box_scale_factors = [.1, .1, .2, .2]
negative_positive_ratio = 3
learning_rate = 1e-3
weight_decay = 5e-4
momentum = 0.9
optimizer = SGD(learning_rate, momentum, decay=weight_decay)
gamma_decay = 0.94
randomize_top = True
weights_path = '../trained_models/VGG16_weights.hdf5'
train_datasets = ['VOC2007', 'VOC2012']
train_splits = ['trainval', 'trainval']
val_dataset = ['VOC2007']
val_split = ['test']
class_names = 'all'
difficult_boxes = True
model_path = '../trained_models/SSD_scratch/'
save_path = model_path + 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'

# data manager
dataset_manager = DataManager(train_datasets, train_splits,
                              class_names, difficult_boxes)
train_data = dataset_manager.load_data()
val_data = test_data = DataManager('VOC2007', 'test').load_data()
class_names = dataset_manager.class_names
num_classes = len(class_names)

# generator
prior_boxes = to_point_form(create_prior_boxes())
generator = ImageGenerator(train_data, val_data, prior_boxes, batch_size,
                           box_scale_factors, num_classes)

# model
multibox_loss = MultiboxLoss(num_classes, negative_positive_ratio, batch_size)
model = SSD300(image_shape, num_classes, weights_path)
model.compile(optimizer, loss=multibox_loss.compute_loss)

# callbacks
if not os.path.exists(model_path):
    os.makedirs(model_path)

checkpoint = ModelCheckpoint(save_path, verbose=1, period=1)
log = CSVLogger(model_path + 'SSD_scratch.log')
learning_rate_manager = LearningRateManager(
                learning_rate, gamma_decay, step_values)
learning_rate_schedule = LearningRateScheduler(learning_rate_manager.schedule)
callbacks = [checkpoint, log, learning_rate_schedule]

# model fit
model.fit_generator(generator.flow(mode='train'),
                    steps_per_epoch=int(len(train_data) / batch_size),
                    epochs=num_epochs,
                    verbose=1,
                    callbacks=callbacks,
                    validation_data=generator.flow(mode='val'),
                    validation_steps=int(len(val_data) / batch_size),
                    # use_multiprocessing=True,
                    max_queue_size=7,
                    workers=5)
<commit_msg>Deleted old training gile<commit_after><|endoftext|>


<commit_before>from __future__ import absolute_import

import os
from sc2reader import utils
from sc2reader import readers
from sc2reader import data
from sc2reader import exceptions
from sc2reader.replay import Replay
from collections import defaultdict

class SC2Reader(object):

    default_options = dict(
        # General use
        verbose=False,
        debug=False,

        # Related to the SC2Reader class only
        register_defaults=True,

        # Related to creating new replay objects
        autoplay=True,
        complete=True,

        # Related to passing paths into load_replay(s)
        directory='',

        # Related to passing a directory to load_replays
        depth=-1,
        exclude=[],
        followlinks=False
    )

    def __init__(self, **options):
        self.registered_readers = defaultdict(list)
        self.registered_datapacks = list()
        self.registered_listeners = defaultdict(list)

        self.options = utils.AttributeDict(utils.merged_dict(self.default_options, options))

        if self.options.register_defaults:
            self.register_defaults()

    def load_replays(self, replay_collection, options=None, **new_options):
        options = options or utils.merged_dict(self.options, new_options)

        # Get the directory and hide it from nested calls
        directory = options.get('directory','')
        if 'directory' in options: del options['directory']

        if isinstance(replay_collection, basestring):
            full_path = os.path.join(directory, replay_collection)
            for replay_path in utils.get_replay_files(full_path, **options):
                with open(replay_path) as replay_file:
                    try:
                        yield self.load_replay(replay_file, options=options)
                    except exceptions.MPQError as e:
                        print e

        else:
            for replay_file in replay_collection:
                if isinstance(replay_file, basestring):
                    full_path = os.path.join(directory, replay_file)
                    if os.path.isdir(full_path):
                        for replay in self.load_replays(full_path, options=options):
                            yield replay
                    else:
                        yield self.load_replay(full_path, options=options)

                else:
                    yield self.load_replay(replay_file, options=options)

    def load_replay(self, replay_file, options=None, **new_options):
        options = options or utils.merged_dict(self.options, new_options)
        complete = options.get('complete',True)
        autoplay = options.get('autoplay',True)

        if isinstance(replay_file, basestring):
            location = os.path.join(options.get('directory',''), replay_file)
            with open(location, 'rb') as replay_file:
                return self.load_replay(replay_file, options=options)

        if options['verbose']:
            print replay_file.name

        replay = Replay(replay_file, **options)

        for data_file in ('replay.initData',
                          'replay.details',
                          'replay.attributes.events',
                          'replay.message.events',):
            reader = self.get_reader(data_file, replay)
            replay.read_data(data_file, reader)

        replay.load_details()
        replay.load_players()

        if complete:
            for data_file in ('replay.game.events',):
                reader = self.get_reader(data_file, replay)
                replay.read_data(data_file, reader)

            replay.load_events(self.get_datapack(replay))

            if autoplay:
                replay.listeners = self.get_listeners(replay)
                replay.start()

        return replay

    def get_reader(self, data_file, replay):
        for callback, reader in self.registered_readers[data_file]:
            if callback(replay):
                return reader

    def get_datapack(self, replay):
        for callback, datapack in self.registered_datapacks:
            if callback(replay):
                return datapack
        return None

    def get_listeners(self, replay):
        listeners = defaultdict(list)
        for event_class in self.registered_listeners.keys():
            for callback, listener in self.registered_listeners[event_class]:
                if callback(replay):
                    listeners[event_class].append(listener)
        return listeners

    def register_listener(self, event_class, listener, callback=lambda r: True):
        self.registered_listeners[event_class].append((callback, listener))

    def register_reader(self, data_file, reader, callback=lambda r: True):
        self.registered_readers[data_file].insert(0,(callback, reader))

    def register_datapack(self, datapack, callback=lambda r: True):
        self.registered_datapacks.insert(0,(callback, datapack))

    def register_defaults(self):
        self.register_default_readers()
        self.register_default_datapacks()
        self.register_default_listeners()

    def register_default_readers(self):
        self.register_reader('replay.details', readers.DetailsReader_Base())
        self.register_reader('replay.initData', readers.InitDataReader_Base())
        self.register_reader('replay.message.events', readers.MessageEventsReader_Base())
        self.register_reader('replay.attributes.events', readers.AttributesEventsReader_Base(), lambda r: r.build <  17326)
        self.register_reader('replay.attributes.events', readers.AttributesEventsReader_17326(), lambda r: r.build >= 17326)
        self.register_reader('replay.game.events', readers.GameEventsReader_Base(), lambda r: r.build <  16561)
        self.register_reader('replay.game.events', readers.GameEventsReader_16561(), lambda r: 16561 <= r.build < 18574)
        self.register_reader('replay.game.events', readers.GameEventsReader_18574(), lambda r: 18574 <= r.build < 19595)
        self.register_reader('replay.game.events', readers.GameEventsReader_19595(), lambda r: 19595 <= r.build)

    def register_default_datapacks(self):
        self.register_datapack(data.Data_16561, lambda r: 16561 <= r.build < 17326)
        self.register_datapack(data.Data_17326, lambda r: 17326 <= r.build < 18317)
        self.register_datapack(data.Data_18317, lambda r: 18317 <= r.build < 19595)
        self.register_datapack(data.Data_19595, lambda r: 19595 <= r.build)

    def register_default_listeners(self):
        #self.register_listener(Event, PrintEventListener())
        pass

    def configure(self, **new_options):
        self.options.update(new_options)

    def reset(self):
        self.options = utils.AttributeDict(self.default_options)

__defaultSC2Reader = SC2Reader()

register_datapack = __defaultSC2Reader.register_datapack
register_listener = __defaultSC2Reader.register_listener
register_reader = __defaultSC2Reader.register_reader

get_listeners = __defaultSC2Reader.get_listeners
get_datapack = __defaultSC2Reader.get_datapack
get_reader = __defaultSC2Reader.get_reader

load_replay = __defaultSC2Reader.load_replay
load_replays = __defaultSC2Reader.load_replays

configure = __defaultSC2Reader.configure
reset = __defaultSC2Reader.reset<commit_msg>Remove some debug code from load_replays..<commit_after>from __future__ import absolute_import

import os
from sc2reader import utils
from sc2reader import readers
from sc2reader import data
from sc2reader import exceptions
from sc2reader.replay import Replay
from collections import defaultdict

class SC2Reader(object):

    default_options = dict(
        # General use
        verbose=False,
        debug=False,

        # Related to the SC2Reader class only
        register_defaults=True,

        # Related to creating new replay objects
        autoplay=True,
        complete=True,

        # Related to passing paths into load_replay(s)
        directory='',

        # Related to passing a directory to load_replays
        depth=-1,
        exclude=[],
        followlinks=False
    )

    def __init__(self, **options):
        self.registered_readers = defaultdict(list)
        self.registered_datapacks = list()
        self.registered_listeners = defaultdict(list)

        self.options = utils.AttributeDict(utils.merged_dict(self.default_options, options))

        if self.options.register_defaults:
            self.register_defaults()

    def load_replays(self, replay_collection, options=None, **new_options):
        options = options or utils.merged_dict(self.options, new_options)

        # Get the directory and hide it from nested calls
        directory = options.get('directory','')
        if 'directory' in options: del options['directory']

        if isinstance(replay_collection, basestring):
            full_path = os.path.join(directory, replay_collection)
            for replay_path in utils.get_replay_files(full_path, **options):
                with open(replay_path) as replay_file:
                    yield self.load_replay(replay_file, options=options)

        else:
            for replay_file in replay_collection:
                if isinstance(replay_file, basestring):
                    full_path = os.path.join(directory, replay_file)
                    if os.path.isdir(full_path):
                        for replay in self.load_replays(full_path, options=options):
                            yield replay
                    else:
                        yield self.load_replay(full_path, options=options)

                else:
                    yield self.load_replay(replay_file, options=options)

    def load_replay(self, replay_file, options=None, **new_options):
        options = options or utils.merged_dict(self.options, new_options)
        complete = options.get('complete',True)
        autoplay = options.get('autoplay',True)

        if isinstance(replay_file, basestring):
            location = os.path.join(options.get('directory',''), replay_file)
            with open(location, 'rb') as replay_file:
                return self.load_replay(replay_file, options=options)

        if options['verbose']:
            print replay_file.name

        replay = Replay(replay_file, **options)

        for data_file in ('replay.initData',
                          'replay.details',
                          'replay.attributes.events',
                          'replay.message.events',):
            reader = self.get_reader(data_file, replay)
            replay.read_data(data_file, reader)

        replay.load_details()
        replay.load_players()

        if complete:
            for data_file in ('replay.game.events',):
                reader = self.get_reader(data_file, replay)
                replay.read_data(data_file, reader)

            replay.load_events(self.get_datapack(replay))

            if autoplay:
                replay.listeners = self.get_listeners(replay)
                replay.start()

        return replay

    def get_reader(self, data_file, replay):
        for callback, reader in self.registered_readers[data_file]:
            if callback(replay):
                return reader

    def get_datapack(self, replay):
        for callback, datapack in self.registered_datapacks:
            if callback(replay):
                return datapack
        return None

    def get_listeners(self, replay):
        listeners = defaultdict(list)
        for event_class in self.registered_listeners.keys():
            for callback, listener in self.registered_listeners[event_class]:
                if callback(replay):
                    listeners[event_class].append(listener)
        return listeners

    def register_listener(self, event_class, listener, callback=lambda r: True):
        self.registered_listeners[event_class].append((callback, listener))

    def register_reader(self, data_file, reader, callback=lambda r: True):
        self.registered_readers[data_file].insert(0,(callback, reader))

    def register_datapack(self, datapack, callback=lambda r: True):
        self.registered_datapacks.insert(0,(callback, datapack))

    def register_defaults(self):
        self.register_default_readers()
        self.register_default_datapacks()
        self.register_default_listeners()

    def register_default_readers(self):
        self.register_reader('replay.details', readers.DetailsReader_Base())
        self.register_reader('replay.initData', readers.InitDataReader_Base())
        self.register_reader('replay.message.events', readers.MessageEventsReader_Base())
        self.register_reader('replay.attributes.events', readers.AttributesEventsReader_Base(), lambda r: r.build <  17326)
        self.register_reader('replay.attributes.events', readers.AttributesEventsReader_17326(), lambda r: r.build >= 17326)
        self.register_reader('replay.game.events', readers.GameEventsReader_Base(), lambda r: r.build <  16561)
        self.register_reader('replay.game.events', readers.GameEventsReader_16561(), lambda r: 16561 <= r.build < 18574)
        self.register_reader('replay.game.events', readers.GameEventsReader_18574(), lambda r: 18574 <= r.build < 19595)
        self.register_reader('replay.game.events', readers.GameEventsReader_19595(), lambda r: 19595 <= r.build)

    def register_default_datapacks(self):
        self.register_datapack(data.Data_16561, lambda r: 16561 <= r.build < 17326)
        self.register_datapack(data.Data_17326, lambda r: 17326 <= r.build < 18317)
        self.register_datapack(data.Data_18317, lambda r: 18317 <= r.build < 19595)
        self.register_datapack(data.Data_19595, lambda r: 19595 <= r.build)

    def register_default_listeners(self):
        #self.register_listener(Event, PrintEventListener())
        pass

    def configure(self, **new_options):
        self.options.update(new_options)

    def reset(self):
        self.options = utils.AttributeDict(self.default_options)

__defaultSC2Reader = SC2Reader()

register_datapack = __defaultSC2Reader.register_datapack
register_listener = __defaultSC2Reader.register_listener
register_reader = __defaultSC2Reader.register_reader

get_listeners = __defaultSC2Reader.get_listeners
get_datapack = __defaultSC2Reader.get_datapack
get_reader = __defaultSC2Reader.get_reader

load_replay = __defaultSC2Reader.load_replay
load_replays = __defaultSC2Reader.load_replays

configure = __defaultSC2Reader.configure
reset = __defaultSC2Reader.reset<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# ff_tools.py
#
# purpose:
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  12-Feb-2012
# modified: Thu 07 Mar 2013 06:07:30 PM BRT
#
# obs:
#

from __future__ import division

import re
import warnings

import numpy as np
import numpy.ma as ma

from netCDF4 import num2date
from dateutil import rrule, parser
from scipy.ndimage import map_coordinates

__all__ = ['wrap_lon180',
           'wrap_lon360',
           'alphanum_key',
           'get_profile',
           'strip_mask',
           'gen_dates',
           'princomp',
           'shiftdim']

def wrap_lon180(lon):
    lon = np.atleast_1d(lon)
    angles = np.logical_or((lon < -180), (180 < lon))
    lon[angles] = wrap_lon360(lon[angles] + 180) - 180
    return lon

def wrap_lon360(lon):
    lon = np.atleast_1d(lon)
    positive = lon > 0
    lon = lon % 360
    lon[np.logical_and(lon == 0, positive)] = 360
    return lon

def alphanum_key(s):
    key = re.split(r"(\d+)", s)
    key[1::2] = map(int, key[1::2])
    return key

def get_profile(x, y, f, xi, yi, mode='nearest', order=3):
    r"""Interpolate regular data.

    Parameters
    ----------
    x : two dimensional np.ndarray
        an array for the :math:`x` coordinates

    y : two dimensional np.ndarray
        an array for the :math:`y` coordinates

    f : two dimensional np.ndarray
        an array with the value of the function to be interpolated
        at :math:`x,y` coordinates.

    xi : one dimension np.ndarray
        the :math:`x` coordinates of the point where we want
        the function to be interpolated.

    yi : one dimension np.ndarray
        the :math:`y` coordinates of the point where we want
        the function to be interpolated.

    order : int
        the order of the bivariate spline interpolation

    Returns
    -------
    fi : one dimension np.ndarray
        the value of the interpolating spline at :math:`xi,yi`

    Examples
    --------
    >>> import numpy as np
    >>> import oceans.ff_tools as ff
    >>> x, y = np.meshgrid(range(360), range(91))
    >>> f = np.array(range(91 * 360)).reshape((91, 360))
    >>> Paris = [2.4, 48.9]
    >>> Rome = [12.5, 41.9]
    >>> Greenwich = [0, 51.5]
    >>> xi = Paris[0], Rome[0], Greenwich[0]
    >>> yi = Paris[1], Rome[1], Greenwich[1]
    >>> ff.get_profile(x, y, f, xi, yi, order=3)

    Notes
    -----
    http://mail.scipy.org/pipermail/scipy-user/2011-June/029857.html
    """

    x, y, f, xi, yi = map(np.asanyarray, (x, y, f, xi, yi))
    conditions = np.array([xi.min() < x.min(),
                           xi.max() > x.max(),
                           yi.min() < y.min(),
                           yi.max() > y.max()])

    if conditions.any():
        warnings.warn("Warning, extrapolation in being done!!")

    dx = x[0, 1] - x[0, 0]
    dy = y[1, 0] - y[0, 0]

    jvals = (xi - x[0, 0]) / dx
    ivals = (yi - y[0, 0]) / dy

    coords = np.array([ivals, jvals])

    return map_coordinates(f, coords, mode=mode, order=order)

def gen_dates(start, end, dt=None):
    r"""Date range from `start` to `end` at `dt` intervals.

    Examples
    --------
    >>> import datetime
    >>> from dateutil import rrule
    >>> start = '1980-01-19'
    >>> end = datetime.datetime.utcnow().strftime("%Y-%m-%d")
    list(gen_dates(start, end, dt=rrule.YEARLY))
    """
    dates = (rrule.rrule(dt, dtstart=parser.parse(start),
                         until=parser.parse(end)))
    return dates

def princomp(A, numpc=None):
    r"""Performs principal components analysis (PCA) on the n-by-p data matrix
    `A`.  Rows of A correspond to observations, columns to variables.

    Returns :
        coeff :
            is a p-by-p matrix, each column containing coefficients
            for one principal component.
        score :
            the principal component scores; that is, the representation
            of A in the principal component space. Rows of SCORE
            correspond to observations, columns to components.
        latent :
        a vector containing the eigenvalues
        of the covariance matrix of A.

    Examples
    --------
    >>> import numpy as np
    >>> import matplotlib.pyplot as plt
    >>> import oceans.ff_tools as ff
    >>> # 2D dataset.
    >>> A = np.array([ [2.4,0.7,2.9,2.2,3.0,2.7,1.6,1.1,1.6,0.9],
                     [2.5,0.5,2.2,1.9,3.1,2.3,2,1,1.5,1.1] ])
    >>> coeff, score, latent = ff.princomp(A.T)
    >>> fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
    >>> # Every eigenvector describe the direction
    >>> # of a principal component.
    >>> m = np.mean(A, axis=1)
    >>> ax1.plot([0, -coeff[0,0] * 2] + m[0], [0, -coeff[0,1] * 2] +
    ...                                                           m[1], '--k')
    >>> ax1.plot([0, coeff[1,0] * 2] + m[0], [0, coeff[1,1] * 2] +
    ...                                                           m[1], '--k')
    >>> ax1.plot(A[0,:], A[1,:], 'ob')  # The data.
    >>> ax1.axis('equal')
    >>> # New data.
    >>> ax2.plot(score[0,:], score[1,:], '*g')
    >>> ax2.axis('equal')
    >>> plt.show()
    >>> # 4D dataset.
    >>> A = np.array([[-1, 1, 2, 2],
    ...               [-2, 3, 1, 0],
    ...               [ 4, 0, 3,-1]], dtype=np.double)
    >>> coeff, score, latent = ff.princomp(A)
    >>> perc = np.cumsum(latent) / np.sum(latent)
    >>> fig, ax = plt.subplots()
    >>> # The following plot show that first two components account for
    >>> # 100% of the variance.
    >>> ax.stem(range(len(perc)), perc, '--b')
    >>> ax.axis([-0.3, 4.3, 0, 1.3])
    >>> plt.show()
    >>> # Image example:
    >>> import matplotlib.cbook as cbook
    >>> from matplotlib.ticker import NullLocator
    >>> img = cbook.get_sample_data('lena.jpg', asfileobj=False)
    >>> A = plt.imread(img) # load an image
    >>> A = np.mean(A, 2)  # to get a 2-D array
    >>> full_pc = np.size(A, axis=1)  # numbers of all the principal components
    >>> i, dist = 1, []
    >>> for numpc in range(0, full_pc+20, 20):
    >>>     coeff, score, latent = ff.princomp(A, numpc)
    >>>     # Reconstruction.difference in Frobenius norm
    >>>     Ar = np.dot(coeff, score).T + np.mean(A, axis=0)
    >>>     dist.append(np.linalg.norm(A - Ar, 'fro'))
    >>>     # Showing the pics reconstructed with less than 50 PCs
    >>>     if numpc <= 50:
    >>>         i += 1
    >>>         print(i)
    >>>         ax = plt.subplot(2, 4, i, frame_on=False)
    >>>         ax.xaxis.set_major_locator(NullLocator())  # remove ticks
    >>>         ax.yaxis.set_major_locator(NullLocator())
    >>>         i += 1
    >>>         plt.imshow(np.flipud(Ar))
    >>>         plt.title('PCs # ' + str(numpc))
    >>>         plt.gray()
    >>> plt.figure()
    >>> plt.imshow(flipud(A))
    >>> plt.title('numpc FULL')
    >>> plt.gray()
    >>> plt.show()
    # At the end of this experiment, we can plot the distance of the
    # reconstructed images from the original image in Frobenius norm
    # (red curve) and the cumulative sum of the eigenvalues (blue curve).
    # Recall that the cumulative sum of the eigenvalues shows the level of
    # variance accounted by each of the corresponding eigenvectors. On the x
    # axis there is the number of eigenvalues/eigenvectors used.
    >>> plt.figure()
    >>> perc = np.cumsum(latent) / np.sum(latent)
    >>> dist = dist / np.max(dist)
    >>> plt.plot(range(len(perc)), perc, 'b',
    >>>          range(0, full_pc + 10, 10), dist, 'r')
    >>> plt.axis([0, full_pc, 0, 1.1])
    >>> plt.show()

    Notes
    -----
    http://glowingpython.blogspot.com/
    2011/07/principal-component-analysis-with-numpy.html?spref=tw
    """

    # computing eigenvalues and eigenvectors of covariance matrix
    M = (A - np.mean(A.T, axis=1)).T  # Subtract the mean along columns.
    [latent, coeff] = np.linalg.eig(np.cov(M))

    if numpc:
        p = np.size(coeff, axis=1)
        idx = np.argsort(latent)  # sorting the eigenvalues
        idx = idx[::-1]  # in ascending order
        # Sorting eigenvectors according to the sorted eigenvalues.
        coeff = coeff[:, idx]
        latent = latent[idx]  # sorting eigenvalues

        if numpc < p or numpc >= 0:
            coeff = coeff[:, range(numpc)]  # Cutting some PCs.

    score = np.dot(coeff.T, M)  # Projection of the data in the new space.
    return coeff, score, latent

def strip_mask(arr, fill_value=np.NaN):
    r"""Take a masked array and return its data(filled) + mask."""
    if ma.isMaskedArray(arr):
        mask = np.ma.getmaskarray(arr)
        arr = np.ma.filled(arr, fill_value)
        return mask, arr
    else:
        return arr

def shiftdim(x, n=None):
    r""" Matlab's shiftdim in python.

    Examples
    --------
    >>> import oceans.ff_tools as ff
    >>> a = np.random.rand(1,1,3,1,2)
    >>> print("a shape and dimension: %s, %s" % (a.shape, a.ndim))
    a shape and dimension: (1, 1, 3, 1, 2), 5
    >>> # print(range(a.ndim))
    >>> # print(np.roll(range(a.ndim), -2))
    >>> # print(a.transpose(np.roll(range(a.ndim), -2)))
    >>> b = ff.shiftdim(a)
    >>> print("b shape and dimension: %s, %s" % (b.shape, b.ndim))
    b shape and dimension: (3, 1, 2), 3
    >>> c = ff.shiftdim(b, -2)
    >>> c.shape == a.shape
    True

    Notes
    -----
    http://www.python-it.org/forum/index.php?topic=4688.0
    """

    def no_leading_ones(shape):
        shape = np.atleast_1d(shape)
        if shape[0] == 1:
            shape = shape[1:]
            return no_leading_ones(shape)
        else:
            return shape

    if n is None:
        # returns the array B with the same number of
        # elements as X but with any leading singleton
        # dimensions removed.
        return x.reshape(no_leading_ones(x.shape))
    elif n >= 0:
        # When n is positive, shiftdim shifts the dimensions
        # to the left and wraps the n leading dimensions to the end.
        return x.transpose(np.roll(range(x.ndim), -n))
    else:
        # When n is negative, shiftdim shifts the dimensions
        # to the right and pads with singletons.
        return x.reshape((1,) * -n + x.shape)

def nc_time(time):
    r"""Convert netcdf time variable to a datetime obj."""
    units = time.units
    calendar = time.calendar
    return num2date(time[:], units, calendar=calendar)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<commit_msg>Fixed examples.<commit_after># -*- coding: utf-8 -*-
#
# ff_tools.py
#
# purpose:
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  12-Feb-2012
# modified: Thu 07 Mar 2013 06:07:30 PM BRT
#
# obs:
#

from __future__ import division

import re
import warnings

import numpy as np
import numpy.ma as ma

from netCDF4 import num2date
from dateutil import rrule, parser
from scipy.ndimage import map_coordinates

__all__ = ['wrap_lon180',
           'wrap_lon360',
           'alphanum_key',
           'get_profile',
           'strip_mask',
           'gen_dates',
           'princomp',
           'shiftdim']

def wrap_lon180(lon):
    lon = np.atleast_1d(lon)
    angles = np.logical_or((lon < -180), (180 < lon))
    lon[angles] = wrap_lon360(lon[angles] + 180) - 180
    return lon

def wrap_lon360(lon):
    lon = np.atleast_1d(lon)
    positive = lon > 0
    lon = lon % 360
    lon[np.logical_and(lon == 0, positive)] = 360
    return lon

def alphanum_key(s):
    key = re.split(r"(\d+)", s)
    key[1::2] = map(int, key[1::2])
    return key

def get_profile(x, y, f, xi, yi, mode='nearest', order=3):
    r"""Interpolate regular data.

    Parameters
    ----------
    x : two dimensional np.ndarray
        an array for the :math:`x` coordinates

    y : two dimensional np.ndarray
        an array for the :math:`y` coordinates

    f : two dimensional np.ndarray
        an array with the value of the function to be interpolated
        at :math:`x,y` coordinates.

    xi : one dimension np.ndarray
        the :math:`x` coordinates of the point where we want
        the function to be interpolated.

    yi : one dimension np.ndarray
        the :math:`y` coordinates of the point where we want
        the function to be interpolated.

    order : int
        the order of the bivariate spline interpolation

    Returns
    -------
    fi : one dimension np.ndarray
        the value of the interpolating spline at :math:`xi,yi`

    Examples
    --------
    >>> import numpy as np
    >>> from oceans.ff_tools import get_profile
    >>> x, y = np.meshgrid(range(360), range(91))
    >>> f = np.array(range(91 * 360)).reshape((91, 360))
    >>> Paris = [2.4, 48.9]
    >>> Rome = [12.5, 41.9]
    >>> Greenwich = [0, 51.5]
    >>> xi = Paris[0], Rome[0], Greenwich[0]
    >>> yi = Paris[1], Rome[1], Greenwich[1]
    >>> get_profile(x, y, f, xi, yi, order=3)

    Notes
    -----
    http://mail.scipy.org/pipermail/scipy-user/2011-June/029857.html
    """

    x, y, f, xi, yi = map(np.asanyarray, (x, y, f, xi, yi))
    conditions = np.array([xi.min() < x.min(),
                           xi.max() > x.max(),
                           yi.min() < y.min(),
                           yi.max() > y.max()])

    if conditions.any():
        warnings.warn("Warning, extrapolation in being done!!")

    dx = x[0, 1] - x[0, 0]
    dy = y[1, 0] - y[0, 0]

    jvals = (xi - x[0, 0]) / dx
    ivals = (yi - y[0, 0]) / dy

    coords = np.array([ivals, jvals])

    return map_coordinates(f, coords, mode=mode, order=order)

def gen_dates(start, end, dt=None):
    r"""Date range from `start` to `end` at `dt` intervals.

    Examples
    --------
    >>> import datetime
    >>> from dateutil import rrule
    >>> start = '1980-01-19'
    >>> end = datetime.datetime.utcnow().strftime("%Y-%m-%d")
    list(gen_dates(start, end, dt=rrule.YEARLY))
    """
    dates = (rrule.rrule(dt, dtstart=parser.parse(start),
                         until=parser.parse(end)))
    return dates

def princomp(A, numpc=None):
    r"""Performs principal components analysis (PCA) on the n-by-p data matrix
    `A`.  Rows of A correspond to observations, columns to variables.

    Returns :
        coeff :
            is a p-by-p matrix, each column containing coefficients
            for one principal component.
        score :
            the principal component scores; that is, the representation
            of A in the principal component space. Rows of SCORE
            correspond to observations, columns to components.
        latent :
        a vector containing the eigenvalues
        of the covariance matrix of A.

    Examples
    --------
    >>> import numpy as np
    >>> import matplotlib.pyplot as plt
    >>> import oceans.ff_tools as ff
    >>> # 2D dataset.
    >>> A = np.array([ [2.4,0.7,2.9,2.2,3.0,2.7,1.6,1.1,1.6,0.9],
                     [2.5,0.5,2.2,1.9,3.1,2.3,2,1,1.5,1.1] ])
    >>> coeff, score, latent = ff.princomp(A.T)
    >>> fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
    >>> # Every eigenvector describe the direction
    >>> # of a principal component.
    >>> m = np.mean(A, axis=1)
    >>> ax1.plot([0, -coeff[0,0] * 2] + m[0], [0, -coeff[0,1] * 2] +
    ...                                                           m[1], '--k')
    >>> ax1.plot([0, coeff[1,0] * 2] + m[0], [0, coeff[1,1] * 2] +
    ...                                                           m[1], '--k')
    >>> ax1.plot(A[0,:], A[1,:], 'ob')  # The data.
    >>> ax1.axis('equal')
    >>> # New data.
    >>> ax2.plot(score[0,:], score[1,:], '*g')
    >>> ax2.axis('equal')
    >>> plt.show()
    >>> # 4D dataset.
    >>> A = np.array([[-1, 1, 2, 2],
    ...               [-2, 3, 1, 0],
    ...               [ 4, 0, 3,-1]], dtype=np.double)
    >>> coeff, score, latent = ff.princomp(A)
    >>> perc = np.cumsum(latent) / np.sum(latent)
    >>> fig, ax = plt.subplots()
    >>> # The following plot show that first two components account for
    >>> # 100% of the variance.
    >>> ax.stem(range(len(perc)), perc, '--b')
    >>> ax.axis([-0.3, 4.3, 0, 1.3])
    >>> plt.show()
    >>> # Image example:
    >>> import matplotlib.cbook as cbook
    >>> from matplotlib.ticker import NullLocator
    >>> img = cbook.get_sample_data('lena.jpg', asfileobj=False)
    >>> A = plt.imread(img) # load an image
    >>> A = np.mean(A, 2)  # to get a 2-D array
    >>> full_pc = np.size(A, axis=1)  # numbers of all the principal components
    >>> i, dist = 1, []
    >>> for numpc in range(0, full_pc+20, 20):
    >>>     coeff, score, latent = ff.princomp(A, numpc)
    >>>     # Reconstruction.difference in Frobenius norm
    >>>     Ar = np.dot(coeff, score).T + np.mean(A, axis=0)
    >>>     dist.append(np.linalg.norm(A - Ar, 'fro'))
    >>>     # Showing the pics reconstructed with less than 50 PCs
    >>>     if numpc <= 50:
    >>>         i += 1
    >>>         print(i)
    >>>         ax = plt.subplot(2, 4, i, frame_on=False)
    >>>         ax.xaxis.set_major_locator(NullLocator())  # remove ticks
    >>>         ax.yaxis.set_major_locator(NullLocator())
    >>>         i += 1
    >>>         plt.imshow(np.flipud(Ar))
    >>>         plt.title('PCs # ' + str(numpc))
    >>>         plt.gray()
    >>> plt.figure()
    >>> plt.imshow(flipud(A))
    >>> plt.title('numpc FULL')
    >>> plt.gray()
    >>> plt.show()
    # At the end of this experiment, we can plot the distance of the
    # reconstructed images from the original image in Frobenius norm
    # (red curve) and the cumulative sum of the eigenvalues (blue curve).
    # Recall that the cumulative sum of the eigenvalues shows the level of
    # variance accounted by each of the corresponding eigenvectors. On the x
    # axis there is the number of eigenvalues/eigenvectors used.
    >>> plt.figure()
    >>> perc = np.cumsum(latent) / np.sum(latent)
    >>> dist = dist / np.max(dist)
    >>> plt.plot(range(len(perc)), perc, 'b',
    >>>          range(0, full_pc + 10, 10), dist, 'r')
    >>> plt.axis([0, full_pc, 0, 1.1])
    >>> plt.show()

    Notes
    -----
    http://glowingpython.blogspot.com/
    2011/07/principal-component-analysis-with-numpy.html?spref=tw
    """

    # computing eigenvalues and eigenvectors of covariance matrix
    M = (A - np.mean(A.T, axis=1)).T  # Subtract the mean along columns.
    [latent, coeff] = np.linalg.eig(np.cov(M))

    if numpc:
        p = np.size(coeff, axis=1)
        idx = np.argsort(latent)  # sorting the eigenvalues
        idx = idx[::-1]  # in ascending order
        # Sorting eigenvectors according to the sorted eigenvalues.
        coeff = coeff[:, idx]
        latent = latent[idx]  # sorting eigenvalues

        if numpc < p or numpc >= 0:
            coeff = coeff[:, range(numpc)]  # Cutting some PCs.

    score = np.dot(coeff.T, M)  # Projection of the data in the new space.
    return coeff, score, latent

def strip_mask(arr, fill_value=np.NaN):
    r"""Take a masked array and return its data(filled) + mask."""
    if ma.isMaskedArray(arr):
        mask = np.ma.getmaskarray(arr)
        arr = np.ma.filled(arr, fill_value)
        return mask, arr
    else:
        return arr

def shiftdim(x, n=None):
    r""" Matlab's shiftdim in python.

    Examples
    --------
    >>> import oceans.ff_tools as ff
    >>> a = np.random.rand(1,1,3,1,2)
    >>> print("a shape and dimension: %s, %s" % (a.shape, a.ndim))
    a shape and dimension: (1, 1, 3, 1, 2), 5
    >>> # print(range(a.ndim))
    >>> # print(np.roll(range(a.ndim), -2))
    >>> # print(a.transpose(np.roll(range(a.ndim), -2)))
    >>> b = ff.shiftdim(a)
    >>> print("b shape and dimension: %s, %s" % (b.shape, b.ndim))
    b shape and dimension: (3, 1, 2), 3
    >>> c = ff.shiftdim(b, -2)
    >>> c.shape == a.shape
    True

    Notes
    -----
    http://www.python-it.org/forum/index.php?topic=4688.0
    """

    def no_leading_ones(shape):
        shape = np.atleast_1d(shape)
        if shape[0] == 1:
            shape = shape[1:]
            return no_leading_ones(shape)
        else:
            return shape

    if n is None:
        # returns the array B with the same number of
        # elements as X but with any leading singleton
        # dimensions removed.
        return x.reshape(no_leading_ones(x.shape))
    elif n >= 0:
        # When n is positive, shiftdim shifts the dimensions
        # to the left and wraps the n leading dimensions to the end.
        return x.transpose(np.roll(range(x.ndim), -n))
    else:
        # When n is negative, shiftdim shifts the dimensions
        # to the right and pads with singletons.
        return x.reshape((1,) * -n + x.shape)

def nc_time(time):
    r"""Convert netcdf time variable to a datetime obj."""
    units = time.units
    calendar = time.calendar
    return num2date(time[:], units, calendar=calendar)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<|endoftext|>