<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import time
import socket

from robot.errors import DataError

from selenium import selenium
from runonfailure import RunOnFailure

BROWSER_ALIASES = {'ff': '*firefox',
                   'firefox': '*firefox',
                   'ie': '*iexplore',
                   'internetexplorer': '*iexplore',
                   'googlechrome': '*googlechrome',
                   'opera': '*opera',
                   'safari': '*safari'}
SELENIUM_CONNECTION_TIMEOUT = 40

class Browser(RunOnFailure):
    """Contains keywords for doing browser actions."""

    def open_browser(self, url, browser='firefox', alias=None):
        """Opens a new browser instance to given URL.

        Possible already opened connections are cached.

        Returns the index of this browser instance which can be used later to
        switch back to it. Index starts from 1 and is reset back to it when
        `Close All Browsers` keyword is used. See `Switch Browser` for
        example.

        Optional alias is a alias for the browser instance and it can be used
        for switching between browsers similarly as the index. See `Switch
        Browser` for more details about that.

        Possible values for `browser` are all the values supported by Selenium
        and some aliases that are defined for convenience. The table below
        lists the aliases for most common supported browsers.

        | firefox          | FireFox   |
        | ff               | FireFox   |
        | ie               | Internet Explorer |
        | internetexplorer | Internet Explorer |
        | safari           | Safari |
        | googlechrome     | Google Chrome |
        | opera            | Opera |

        Additionally, a string like `*custom /path/to/browser-executable` can
        be used to specify the browser directly. In this case, the path needs to
        point to an executable, not a script, otherwise the library may not be
        able to shut down the browser properly.

        Note, that you will encounter strange behavior, if you open
        multiple Internet Explorer browser instances. That is also why
        `Switch Browser` only works with one IE browser at most.
        For more information see:
        http://selenium-grid.seleniumhq.org/faq.html#i_get_some_strange_errors_when_i_run_multiple_internet_explorer_instances_on_the_same_machine
        """
        self._info("Opening browser '%s' to base url '%s'" % (browser, url))
        browser = self._get_browser(browser)
        self._selenium = selenium(self._server_host, self._server_port, browser,
                                  url)
        self._connect_to_selenium_server()
        self._selenium.set_timeout(self._timeout * 1000)
        self._selenium.open(url, ignoreResponseCode=True)
        self._debug('Opened browser with Selenium session id %s'
                    % self._selenium.sessionId)
        return self._cache.register(self._selenium, alias)

    def _get_browser(self, browser):
        return BROWSER_ALIASES.get(browser.lower().replace(' ', ''), browser)

    def _connect_to_selenium_server(self):
        timeout = time.time() + SELENIUM_CONNECTION_TIMEOUT
        while time.time() < timeout:
            try:
                self._selenium.start()
            # AssertionError occurs on Jython: http://bugs.jython.org/issue1697
            except (socket.error, AssertionError):
                time.sleep(2)
            else:
                return
        self._selenium = NoBrowserOpen()
        raise RuntimeError("Could not connect to Selenium Server in %d seconds. "
                           "Please make sure Selenium Server is running."
                           % SELENIUM_CONNECTION_TIMEOUT)

    def close_browser(self):
        """Closes the current browser."""
        if self._selenium:
            self._debug('Closing browser with Selenium session id %s'
                        % self._selenium.sessionId)
            self.close_window()
            self._selenium.stop()
            self._cache.current = None
            self._selenium = NoBrowserOpen()

    def close_all_browsers(self):
        """Closes all open browsers and empties the connection cache.

        After this keyword new indexes get from Open Browser keyword are reset
        to 1.

        This keyword should be used in test or suite teardown to make sure
        all browsers are closed.
        """
        # ConnectionCache's connections attribute was renamed to _connections
        # in RF 2.0.2 (which was actually a stupid decision)
        try:
            connections = self._cache._connections
        except AttributeError:
            connections = self._cache.connections
        for sel in connections:
            if sel is not None and sel.sessionId is not None:
                self._debug('Closing browser with Selenium session id %s'
                            % sel.sessionId)
                sel.stop()
        self._selenium = NoBrowserOpen()
        self._cache.empty_cache()

    def switch_browser(self, index_or_alias):
        """Switches between active browsers using index or alias.

        Index is got from `Open Browser` and alias can be given to it.

        Examples:
        | Open Browser        | http://google.com | ff       |
        | Location Should Be  | http://google.com |          |
        | Open Browser        | http://yahoo.com  | ie       | 2nd conn |
        | Location Should Be  | http://yahoo.com  |          |
        | Switch Browser      | 1                 | # index  |
        | Page Should Contain | I'm feeling lucky |          |
        | Switch Browser      | 2nd conn          | # alias  |
        | Page Should Contain | More Yahoo!       |          |
        | Close All Browsers  |                   |          |

        Above example expects that there was no other open browsers when
        opening the first one because it used index '1' when switching to it
        later. If you aren't sure about that you can store the index into
        a variable as below.

        | ${id} =            | Open Browser  | http://google.com | *firefox |
        | # Do something ... |
        | Switch Browser     | ${id}         |                   |          |
        """
        try:
            self._selenium = self._cache.switch(index_or_alias)
            self._debug('Switched to browser with Selenium session id %s'
                         % self._selenium.sessionId)
        except (RuntimeError, DataError):  # RF 2.6 uses RE, earlier DE
            raise RuntimeError("No browser with index or alias '%s' found."
                               % index_or_alias)

    def go_to(self, url):
        """Navigates the active browser instance to the provided URL."""
        self._info("Opening url '%s'" % url)
        self._selenium.open(url, ignoreResponseCode=True)

    def wait_until_page_loaded(self, timeout=None):
        """Waits for a page load to happen.

        This keyword can be used after performing an action that causes a page
        load to ensure that following keywords see the page fully loaded.

        `timeout` is the time to wait for the page load to happen, after which
        this keyword fails. If `timeout` is not provided, the value given in
        `importing` or using keyword `Set Timeout` is used.

        Many of the keywords that cause a page load take an optional argument
        `dont_wait` that can be also used to wait/not wait page load. See
        `introduction` for more details.

        This keyword was added in SeleniumLibrary 2.5.
        """
        timeout = self._get_timeout(timeout)
        self._selenium.wait_for_page_to_load(timeout * 1000)

    def reload_page(self):
        """Simulates user reloading page.

        New in SeleniumLibrary 2.7.
        """
        self._selenium.refresh()
        self.wait_until_page_loaded()

    def go_back(self, dont_wait=''):
        """Simulates the user clicking the "back" button on their browser.

        See `introduction` for details about locating elements and about meaning
        of `dont_wait` argument."""
        self._selenium.go_back()
        if not dont_wait:
            self.wait_until_page_loaded()

    def maximize_browser_window(self):
        """Maximizes current browser window."""
        self._selenium.window_maximize()

    def get_window_names(self):
        """Returns and logs names of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_names(), 'name')

    def get_window_titles(self):
        """Returns and logs titles of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_titles(), 'title')

    def get_window_identifiers(self):
        """Returns and logs id attributes of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_ids(), 'identifier')

    def select_window(self, locator='main'):
        """Selects the window found with `locator` as the context of actions.

        If the window is found, all subsequent commands use that window, until
        this keyword is used again. If the window is not found, this keyword fails.

        `locator` may be either the title of the window or the name of the window
        in the JavaScript code that creates it. If multiple windows with
        same identifier are found, the first one is selected.

        Special locator `main` (default) can be used to select the main window.

        Example:
        | Click Link | popup_link | don't wait | # opens new window |
        | Select Window | popupName |
        | Title Should Be | Popup Title |
        | Select Window |  | | # Chooses the main window again |

        *NOTE:* Selecting windows opened by links with target `_blank` does
        not seem to work on Internet Explorer.
        """
        if locator.lower() == 'main':
            locator = 'null'
        self._selenium.select_window(locator)

    def close_window(self):
        """Closes currently opened pop-up window."""
        self._selenium.close()

    def get_location(self):
        """Returns the current location."""
        return self._selenium.get_location()

    def get_cookies(self):
        """Returns all cookies of the current page."""
        return self._selenium.get_cookie()

    def get_cookie_value(self, name):
        """Returns value of cookie found with `name`.

        If no cookie is found with `name`, this keyword fails.
        """
        return self._selenium.get_cookie_by_name(name)

    def delete_cookie(self, name, options=''):
        """Deletes cookie matching `name` and `options`.

        If the cookie is not found, nothing happens.

        `options` is the options for the cookie as a string. Currently
        supported options include `path`, `domain` and `recurse.` Format for
        `options` is `path=/path/, domain=.foo.com, recurse=true`. The order of
        options is irrelevant. Note that specifying a domain that is not a
        subset of the current domain will usually fail. Setting `recurse=true`
        will cause this keyword to search all sub-domains of current domain
        with all paths that are subset of current path. This can take a long
        time.
        """
        self._selenium.delete_cookie(name, options)

    def delete_all_cookies(self):
        """Deletes all cookies by calling `Delete Cookie` repeatedly."""
        self._selenium.delete_all_visible_cookies()

class NoBrowserOpen(object):
    set_timeout = lambda self, timeout: None

    def __getattr__(self, name):
        raise RuntimeError('No browser is open')

    def __nonzero__(self):
        return False
<commit_msg>browser: use --disable-web-security always with googlechrome<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import time
import socket

from robot.errors import DataError

from selenium import selenium
from runonfailure import RunOnFailure

BROWSER_ALIASES = {'ff': '*firefox',
                   'firefox': '*firefox',
                   'ie': '*iexplore',
                   'internetexplorer': '*iexplore',
                   'googlechrome': '*googlechrome',
                   'opera': '*opera',
                   'safari': '*safari'}
SELENIUM_CONNECTION_TIMEOUT = 40

class Browser(RunOnFailure):
    """Contains keywords for doing browser actions."""

    def open_browser(self, url, browser='firefox', alias=None):
        """Opens a new browser instance to given URL.

        Possible already opened connections are cached.

        Returns the index of this browser instance which can be used later to
        switch back to it. Index starts from 1 and is reset back to it when
        `Close All Browsers` keyword is used. See `Switch Browser` for
        example.

        Optional alias is a alias for the browser instance and it can be used
        for switching between browsers similarly as the index. See `Switch
        Browser` for more details about that.

        Possible values for `browser` are all the values supported by Selenium
        and some aliases that are defined for convenience. The table below
        lists the aliases for most common supported browsers.

        | firefox          | FireFox   |
        | ff               | FireFox   |
        | ie               | Internet Explorer |
        | internetexplorer | Internet Explorer |
        | safari           | Safari |
        | googlechrome     | Google Chrome |
        | opera            | Opera |

        Additionally, a string like `*custom /path/to/browser-executable` can
        be used to specify the browser directly. In this case, the path needs to
        point to an executable, not a script, otherwise the library may not be
        able to shut down the browser properly.

        Note, that you will encounter strange behavior, if you open
        multiple Internet Explorer browser instances. That is also why
        `Switch Browser` only works with one IE browser at most.
        For more information see:
        http://selenium-grid.seleniumhq.org/faq.html#i_get_some_strange_errors_when_i_run_multiple_internet_explorer_instances_on_the_same_machine
        """
        self._info("Opening browser '%s' to base url '%s'" % (browser, url))
        browser = self._get_browser(browser)
        self._selenium = selenium(self._server_host, self._server_port, browser,
                                  url)
        self._connect_to_selenium_server(self._browser_options(browser))
        self._selenium.set_timeout(self._timeout * 1000)
        self._selenium.open(url, ignoreResponseCode=True)
        self._debug('Opened browser with Selenium session id %s'
                    % self._selenium.sessionId)
        return self._cache.register(self._selenium, alias)

    def _get_browser(self, browser):
        return BROWSER_ALIASES.get(browser.lower().replace(' ', ''), browser)

    def _browser_options(self, browser):
        if browser == '*googlechrome':
            return 'commandLineFlags=--disable-web-security'
        return ''

    def _connect_to_selenium_server(self, options):
        timeout = time.time() + SELENIUM_CONNECTION_TIMEOUT
        while time.time() < timeout:
            try:
                self._selenium.start(options)
            # AssertionError occurs on Jython: http://bugs.jython.org/issue1697
            except (socket.error, AssertionError):
                time.sleep(2)
            else:
                return
        self._selenium = NoBrowserOpen()
        raise RuntimeError("Could not connect to Selenium Server in %d seconds. "
                           "Please make sure Selenium Server is running."
                           % SELENIUM_CONNECTION_TIMEOUT)

    def close_browser(self):
        """Closes the current browser."""
        if self._selenium:
            self._debug('Closing browser with Selenium session id %s'
                        % self._selenium.sessionId)
            self.close_window()
            self._selenium.stop()
            self._cache.current = None
            self._selenium = NoBrowserOpen()

    def close_all_browsers(self):
        """Closes all open browsers and empties the connection cache.

        After this keyword new indexes get from Open Browser keyword are reset
        to 1.

        This keyword should be used in test or suite teardown to make sure
        all browsers are closed.
        """
        # ConnectionCache's connections attribute was renamed to _connections
        # in RF 2.0.2 (which was actually a stupid decision)
        try:
            connections = self._cache._connections
        except AttributeError:
            connections = self._cache.connections
        for sel in connections:
            if sel is not None and sel.sessionId is not None:
                self._debug('Closing browser with Selenium session id %s'
                            % sel.sessionId)
                sel.stop()
        self._selenium = NoBrowserOpen()
        self._cache.empty_cache()

    def switch_browser(self, index_or_alias):
        """Switches between active browsers using index or alias.

        Index is got from `Open Browser` and alias can be given to it.

        Examples:
        | Open Browser        | http://google.com | ff       |
        | Location Should Be  | http://google.com |          |
        | Open Browser        | http://yahoo.com  | ie       | 2nd conn |
        | Location Should Be  | http://yahoo.com  |          |
        | Switch Browser      | 1                 | # index  |
        | Page Should Contain | I'm feeling lucky |          |
        | Switch Browser      | 2nd conn          | # alias  |
        | Page Should Contain | More Yahoo!       |          |
        | Close All Browsers  |                   |          |

        Above example expects that there was no other open browsers when
        opening the first one because it used index '1' when switching to it
        later. If you aren't sure about that you can store the index into
        a variable as below.

        | ${id} =            | Open Browser  | http://google.com | *firefox |
        | # Do something ... |
        | Switch Browser     | ${id}         |                   |          |
        """
        try:
            self._selenium = self._cache.switch(index_or_alias)
            self._debug('Switched to browser with Selenium session id %s'
                         % self._selenium.sessionId)
        except (RuntimeError, DataError):  # RF 2.6 uses RE, earlier DE
            raise RuntimeError("No browser with index or alias '%s' found."
                               % index_or_alias)

    def go_to(self, url):
        """Navigates the active browser instance to the provided URL."""
        self._info("Opening url '%s'" % url)
        self._selenium.open(url, ignoreResponseCode=True)

    def wait_until_page_loaded(self, timeout=None):
        """Waits for a page load to happen.

        This keyword can be used after performing an action that causes a page
        load to ensure that following keywords see the page fully loaded.

        `timeout` is the time to wait for the page load to happen, after which
        this keyword fails. If `timeout` is not provided, the value given in
        `importing` or using keyword `Set Timeout` is used.

        Many of the keywords that cause a page load take an optional argument
        `dont_wait` that can be also used to wait/not wait page load. See
        `introduction` for more details.

        This keyword was added in SeleniumLibrary 2.5.
        """
        timeout = self._get_timeout(timeout)
        self._selenium.wait_for_page_to_load(timeout * 1000)

    def reload_page(self):
        """Simulates user reloading page.

        New in SeleniumLibrary 2.7.
        """
        self._selenium.refresh()
        self.wait_until_page_loaded()

    def go_back(self, dont_wait=''):
        """Simulates the user clicking the "back" button on their browser.

        See `introduction` for details about locating elements and about meaning
        of `dont_wait` argument."""
        self._selenium.go_back()
        if not dont_wait:
            self.wait_until_page_loaded()

    def maximize_browser_window(self):
        """Maximizes current browser window."""
        self._selenium.window_maximize()

    def get_window_names(self):
        """Returns and logs names of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_names(), 'name')

    def get_window_titles(self):
        """Returns and logs titles of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_titles(), 'title')

    def get_window_identifiers(self):
        """Returns and logs id attributes of all windows known to the browser."""
        return self._log_list(self._selenium.get_all_window_ids(), 'identifier')

    def select_window(self, locator='main'):
        """Selects the window found with `locator` as the context of actions.

        If the window is found, all subsequent commands use that window, until
        this keyword is used again. If the window is not found, this keyword fails.

        `locator` may be either the title of the window or the name of the window
        in the JavaScript code that creates it. If multiple windows with
        same identifier are found, the first one is selected.

        Special locator `main` (default) can be used to select the main window.

        Example:
        | Click Link | popup_link | don't wait | # opens new window |
        | Select Window | popupName |
        | Title Should Be | Popup Title |
        | Select Window |  | | # Chooses the main window again |

        *NOTE:* Selecting windows opened by links with target `_blank` does
        not seem to work on Internet Explorer.
        """
        if locator.lower() == 'main':
            locator = 'null'
        self._selenium.select_window(locator)

    def close_window(self):
        """Closes currently opened pop-up window."""
        self._selenium.close()

    def get_location(self):
        """Returns the current location."""
        return self._selenium.get_location()

    def get_cookies(self):
        """Returns all cookies of the current page."""
        return self._selenium.get_cookie()

    def get_cookie_value(self, name):
        """Returns value of cookie found with `name`.

        If no cookie is found with `name`, this keyword fails.
        """
        return self._selenium.get_cookie_by_name(name)

    def delete_cookie(self, name, options=''):
        """Deletes cookie matching `name` and `options`.

        If the cookie is not found, nothing happens.

        `options` is the options for the cookie as a string. Currently
        supported options include `path`, `domain` and `recurse.` Format for
        `options` is `path=/path/, domain=.foo.com, recurse=true`. The order of
        options is irrelevant. Note that specifying a domain that is not a
        subset of the current domain will usually fail. Setting `recurse=true`
        will cause this keyword to search all sub-domains of current domain
        with all paths that are subset of current path. This can take a long
        time.
        """
        self._selenium.delete_cookie(name, options)

    def delete_all_cookies(self):
        """Deletes all cookies by calling `Delete Cookie` repeatedly."""
        self._selenium.delete_all_visible_cookies()

class NoBrowserOpen(object):
    set_timeout = lambda self, timeout: None

    def __getattr__(self, name):
        raise RuntimeError('No browser is open')

    def __nonzero__(self):
        return False
<|endoftext|>


<commit_before>"""
Visualize networks via results on axes
"""

import sys
import pickle

import numpy as np
import pandas as pd

import seaborn as sns
import matplotlib as mpl
import matplotlib.pylab as plt

from utils import extract_sig_entries
from plotter import save_figure
from network_matrix import plot_individuals

def transform(data):
    """ Convert input data into DataFrame
    """
    df = pd.DataFrame()
    for raw_res, enh_res_vec in data:
        df = df.append(
            [{
                'raw_res': raw_res,
                'enh_res': enh_res,
                'raw_mat': raw_res[1],
                'enh_mat': enh_res[1]
            } for enh_res in enh_res_vec],
            ignore_index=True)
    return df[~df.isnull().any(axis=1)]

def extract_entries(row):
    """ Extract needed amount of entries from each matrix
    """
    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'raw_vals': extract_sig_entries(row.raw_mat),
        'enh_vals': extract_sig_entries(row.enh_mat[:3,:3])
            if not row.enh_mat is None else None
    })

def bin_entries(row, threshold=0.2):
    """ Bin entries
    """
    def rm(vals):
        vals[abs(vals) < threshold] = 0
        return vals

    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'raw_vals': rm(row.raw_vals),
        'enh_vals': rm(row.enh_vals)
    })

def check_vanishing(row):
    """ Estimate number of vanishing correlations
    """
    mask = np.zeros(row.raw_vals.shape, dtype=bool)
    nv_inds = np.intersect1d(np.nonzero(row.raw_vals), np.nonzero(row.enh_vals))
    mask[nv_inds] = 1

    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'already_zero': np.sum(row.raw_vals == 0),
        'abs_corr_diff': np.mean(abs(row.raw_vals[mask] - row.enh_vals[mask]))
            if len(nv_inds) > 0 else np.nan
    })

def check_neg_pos_corrs(row):
    neg_pos = []
    pos_neg = []

    for raw, enh in zip(row.raw_vals, row.enh_vals):
        if raw < 0:
            neg_pos.append(abs(raw-enh))
        else:
            pos_neg.append(abs(raw-enh))

    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'mean_negpos_corr': np.mean(neg_pos),
        'mean_posneg_corr': np.mean(pos_neg)
    })

def check_disruption(row):
    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'non_trivial_disruption': 1 - np.mean(abs(row.enh_vals)),
        'mean_disruption': np.mean(abs(row.raw_vals - row.enh_vals))
    })

def plot_zeros_vs_corr_diff(df):
    """ Plot number of zero correlations in original network against mean of absolute correlation differences
    """
    df = df.apply(check_vanishing, axis=1)

    # generate overview
    plt.figure()

    sns.regplot(
        x='already_zero', y='abs_corr_diff', data=df,
        fit_reg=False,
        x_jitter=0.1)#, y_jitter=0.1)

    save_figure('images/network_space.pdf')

    # plot networks in detail
    extr = df[df.abs_corr_diff > 0.4]
    print(extr)

    if not extr.empty:
        mpl.style.use('default')
        plot_individuals(
            list(zip(extr.raw_res.tolist(), extr.enh_res.tolist())),
            'images/extreme.pdf')

def plot_neg_pos_corrs(df):
    """ Plot neg - pos corrs on x and pos - neg corrs on y axis
    """
    df = df.apply(check_neg_pos_corrs, axis=1)

    # generate overview
    plt.figure()
    sns.regplot(
        x='mean_negpos_corr', y='mean_posneg_corr', data=df,
        fit_reg=False)
    save_figure('images/network_space2.pdf')

    # plot networks in detail
    extr = df[(df.mean_negpos_corr > 0.6) & (df.mean_posneg_corr > 0.6)]
    print(extr)

    if not extr.empty:
        mpl.style.use('default')
        plot_individuals(
            list(zip(extr.raw_res.tolist(), extr.enh_res.tolist()))[:5],
            'images/extreme2.pdf')

def plot_disruptiveness(df):
    """ Compare disruption of correlations (mean difference) to trivial disruption (1 mean(corr. of 4node network))
    """
    df = df.apply(check_disruption, axis=1)

    # generate overview
    plt.figure()
    sns.regplot(
        x='non_trivial_disruption', y='mean_disruption', data=df,
        fit_reg=False)
    save_figure('images/network_space2.pdf')

    # plot networks in detail
    extr = df[(df.non_trivial_disruption >= 0.6) & (df.mean_disruption >= 0.3)]
    print(extr)

    if not extr.empty:
        mpl.style.use('default')
        plot_individuals(
            list(zip(extr.raw_res.tolist(), extr.enh_res.tolist())),
            'images/extreme2.pdf')

def main(data):
    """ Analyse data
    """
    df = transform(data)
    df = df.apply(
        lambda row: bin_entries(extract_entries(row)),
        axis=1)

    #plot_neg_pos_corrs(df)
    #plot_zeros_vs_corr_diff(df)
    plot_disruptiveness(df)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        fname = sys.argv[1]
        with open(fname, 'rb') as fd:
            inp = pickle.load(fd)
        main(inp['data'])
    else:
        print('Usage: {} <data file>'.format(sys.argv[0]))
        exit(1)
<commit_msg>Tweak investigation<commit_after>"""
Visualize networks via results on axes
"""

import sys
import pickle

import numpy as np
import pandas as pd

import seaborn as sns
import matplotlib as mpl
import matplotlib.pylab as plt

from utils import extract_sig_entries
from plotter import save_figure
from network_matrix import plot_individuals

def transform(data):
    """ Convert input data into DataFrame
    """
    df = pd.DataFrame()
    for raw_res, enh_res_vec in data:
        df = df.append(
            [{
                'raw_res': raw_res,
                'enh_res': enh_res,
                'raw_mat': raw_res[1],
                'enh_mat': enh_res[1]
            } for enh_res in enh_res_vec],
            ignore_index=True)
    return df[~df.isnull().any(axis=1)]

def extract_entries(row):
    """ Extract needed amount of entries from each matrix
    """
    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'raw_vals': extract_sig_entries(row.raw_mat),
        'enh_vals': extract_sig_entries(row.enh_mat[:3,:3])
            if not row.enh_mat is None else None
    })

def bin_entries(row, threshold=0.2):
    """ Bin entries
    """
    def rm(vals):
        vals[abs(vals) < threshold] = 0
        return vals

    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'raw_vals': rm(row.raw_vals),
        'enh_vals': rm(row.enh_vals)
    })

def check_disruption(row):

    sdiff = (np.sign(row.raw_vals) - np.sign(row.enh_vals)) * abs(row.raw_vals - row.enh_vals)

    return pd.Series({
        'raw_res': row.raw_res,
        'enh_res': row.enh_res,
        'sign_diff': np.mean(sdiff),
        'mean_difference': np.mean(abs(row.raw_vals - row.enh_vals))
    })

def plot_disruptiveness(df):
    """ Compare disruption of correlations (mean difference) to trivial disruption (1 mean(corr. of 4node network))
    """
    df = df.apply(check_disruption, axis=1)

    # generate overview
    plt.figure()
    sns.regplot(
        x='sign_diff', y='mean_difference', data=df,
        fit_reg=False)
    save_figure('images/network_space.pdf')

    # plot networks in detail
    extr = df[(df.sign_diff >= 0.3) & (df.mean_difference >= 0.4)]
    print(extr)

    if not extr.empty:
        mpl.style.use('default')
        plot_individuals(
            list(zip(extr.raw_res.tolist(), extr.enh_res.tolist())),
            'images/extreme.pdf')

def main(data):
    """ Analyse data
    """
    df = transform(data)
    df = df.apply(
        lambda row: bin_entries(extract_entries(row)),
        axis=1)

    plot_disruptiveness(df)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        fname = sys.argv[1]
        with open(fname, 'rb') as fd:
            inp = pickle.load(fd)
        main(inp['data'])
    else:
        print('Usage: {} <data file>'.format(sys.argv[0]))
        exit(1)
<|endoftext|>


<commit_before>"""
Wrapper for thermal functions from C++ library with SWIG
========================================================

There are various implmentations of the thermal functions J_F(y^2) and J_B(y^2),
and their first- and second-derivatives.

They should be called via

>>> J_B(100., method='quad')
-0.0021509975248131265

>>> J_F(100., method='bessel')
0.002150965876834246

The method is an optional argument. First and second derivatives are
implemented by a keyword argument:

>>> J_F(100., derivative=1)
-9.324239650463552e-05

Tests:

>>> import numpy as np
>>> y_squared = np.linspace(-10, 10, 10)

>>> for method in METHODS:
...     np.vectorize(J_B)(y_squared, method)
...     np.vectorize(J_F)(y_squared, method)
array([-1.03301551, -2.37643967, -3.22920627, -3.4586863 , -2.87437545,
       -1.65904686, -1.13857283, -0.83787385, -0.63967438, -0.50041956])
array([ 2.55378622,  4.62466887,  4.57599995,  3.66746647,  2.44541129,
        1.53766395,  1.0938664 ,  0.81725016,  0.62904789,  0.49454272])
array([-1.03301518, -2.37643835, -3.22920637, -3.45867784, -2.87436571,
       -1.65904681, -1.13857282, -0.83787385, -0.63967438, -0.50041954])
array([ 2.55328922,  4.62461661,  4.57599754,  3.66745192,  2.44540827,
        1.53766392,  1.0938664 ,  0.81725016,  0.62904789,  0.4945427 ])
array([-9.7084274 , -7.62164949, -5.90432495, -4.42150614, -2.98134396,
       -1.7660153 , -2.10137679, -3.51278671, -5.88376321, -9.17182574])
array([ 15.43326144,  11.3026159 ,   7.6896565 ,   4.71314561,
         2.55512445,   1.64214217,   1.99504072,   3.22861748,
         5.20114362,   7.83237021])
array([ 8.2744023 ,  6.85296131,  5.32454196,  3.62990861,  1.59240946,
        1.59240946,  3.62990861,  5.32454196,  6.85296131,  8.2744023 ])
array([ 8.2744023 ,  6.85296131,  5.32454196,  3.62990861,  1.59240946,
        1.59240946,  3.62990861,  5.32454196,  6.85296131,  8.2744023 ])
array([-4.87947524, -5.29925691, -4.53529364, -2.66696586, -0.36007938,
       -0.47270639, -0.49809179, -0.42950048, -0.35893197, -0.29833225])
array([ 4.87947524,  5.29925691,  4.53529364,  2.66696586,  0.36007938,
        0.47270639,  0.49809179,  0.42950048,  0.35893197,  0.29833225])
array([-4.24227668, -4.52683182, -4.01645497, -2.66394847, -0.61658766,
       -0.47270639, -0.49809179, -0.42950048, -0.35893197, -0.29833225])
array([ 6.41771861,  6.83162332,  4.9203404 ,  2.34920927,  0.18597062,
        0.44674405,  0.48467656,  0.42254647,  0.35511487,  0.29613348])

Test derivative against numerical derivatives.

>>> np.vectorize(J_F)(y_squared, derivative=1, method="bessel") - np.vectorize(J_F)(y_squared, derivative=1, method="approx")
array([ -5.06825245e-03,   8.86800478e-05,   2.29852685e-05,
        -3.26269292e-04,  -2.71182328e-04,  -1.72205933e-07,
        -8.12912029e-08,   1.40641963e-07,  -4.34322898e-08,
         2.84767146e-08])

>>> np.vectorize(J_F)(y_squared, derivative=2, method="bessel") - np.vectorize(J_F)(y_squared, derivative=2, method="approx")
    array([ -5.28890689e-01,   4.34476713e-04,   2.28543869e-02,
             8.60189656e-03,   3.08677351e-02,   3.05994751e-02,
             1.99819211e-03,   4.25091227e-04,   1.31940097e-04,
             4.99716208e-05])

>>> np.vectorize(J_B)(y_squared, derivative=1, method="bessel") - np.vectorize(J_B)(y_squared, derivative=1, method="approx")
array([ -2.16951985e-04,  -2.55114955e-04,   6.97032234e-05,
        -3.38789396e-04,  -2.70543819e-04,   3.85995679e-07,
         1.32647813e-08,   1.30533110e-07,   1.23921645e-07,
        -3.70553466e-08])

>>> np.vectorize(J_B)(y_squared, derivative=2, method="bessel") - np.vectorize(J_B)(y_squared, derivative=2, method="approx")
array([ -1.18211875e-02,  -1.07143055e-02,  -2.78539577e-03,
         1.84933021e-02,   1.84354041e-01,   5.28891196e-02,
         2.65086492e-03,   5.05682480e-04,   1.48106517e-04,
         5.41656245e-05])
"""

# Load library

import _thermal_funcs

METHODS = ['quad', 'bessel', 'taylor', 'lim', 'approx', 'zeta']
METHODS_DERIV = ['bessel', 'approx']

def J_F(y_squared, method="bessel", derivative=0, **kwargs):
    """
    :param y_squared: Thermal function argument
    :type y_squared: float
    :param method: Method of calculating thermal function
    :type method: str
    :param derivative: Order of deriviatve
    :type derivative: int

    :returns: Thermal function J_F(y^2)
    :rtype: float
    """
    if derivative == 0:
        assert method in METHODS, "{} is not a known method: {}".format(method, METHODS)
        name = "J_F_{}".format(method)
    elif derivative == 1:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D1_J_F_{}".format(method)
    elif derivative == 2:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D2_J_F_{}".format(method)
    else:
        raise RuntimeError("Order must be 0 <= derivative <= 2")

    func = getattr(_thermal_funcs, name)
    return func(y_squared, **kwargs)

def J_B(y_squared, method="bessel", derivative=0, **kwargs):
    """
    :param y_squared: Thermal function argument
    :type y_squared: float
    :param method: Method of calculating thermal function
    :type method: str
    :param derivative: Order of deriviatve
    :type derivative: int

    :returns: Thermal function J_B(y^2)
    :rtype: float
    """
    if derivative == 0:
        assert method in METHODS, "{} is not a known method: {}".format(method, METHODS)
        name = "J_B_{}".format(method)
    elif derivative == 1:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D1_J_B_{}".format(method)
    elif derivative == 2:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D2_J_B_{}".format(method)
    else:
        raise RuntimeError("Order must be 0 <= derivative <= 2")

    func = getattr(_thermal_funcs, name)
    return func(y_squared, **kwargs)

if __name__ == '__main__':
    import doctest
    doctest.testmod(optionflags=doctest.REPORT_NDIFF)
    doctest.testmod()

<commit_msg>remove trailing whitespace<commit_after>"""
Wrapper for thermal functions from C++ library with SWIG
========================================================

There are various implmentations of the thermal functions J_F(y^2) and J_B(y^2),
and their first- and second-derivatives.

They should be called via

>>> J_B(100., method='quad')
-0.0021509975248131265

>>> J_F(100., method='bessel')
0.002150965876834246

The method is an optional argument. First and second derivatives are
implemented by a keyword argument:

>>> J_F(100., derivative=1)
-9.324239650463552e-05

Tests:

>>> import numpy as np
>>> y_squared = np.linspace(-10, 10, 10)

>>> for method in METHODS:
...     np.vectorize(J_B)(y_squared, method)
...     np.vectorize(J_F)(y_squared, method)
array([-1.03301551, -2.37643967, -3.22920627, -3.4586863 , -2.87437545,
       -1.65904686, -1.13857283, -0.83787385, -0.63967438, -0.50041956])
array([ 2.55378622,  4.62466887,  4.57599995,  3.66746647,  2.44541129,
        1.53766395,  1.0938664 ,  0.81725016,  0.62904789,  0.49454272])
array([-1.03301518, -2.37643835, -3.22920637, -3.45867784, -2.87436571,
       -1.65904681, -1.13857282, -0.83787385, -0.63967438, -0.50041954])
array([ 2.55328922,  4.62461661,  4.57599754,  3.66745192,  2.44540827,
        1.53766392,  1.0938664 ,  0.81725016,  0.62904789,  0.4945427 ])
array([-9.7084274 , -7.62164949, -5.90432495, -4.42150614, -2.98134396,
       -1.7660153 , -2.10137679, -3.51278671, -5.88376321, -9.17182574])
array([ 15.43326144,  11.3026159 ,   7.6896565 ,   4.71314561,
         2.55512445,   1.64214217,   1.99504072,   3.22861748,
         5.20114362,   7.83237021])
array([ 8.2744023 ,  6.85296131,  5.32454196,  3.62990861,  1.59240946,
        1.59240946,  3.62990861,  5.32454196,  6.85296131,  8.2744023 ])
array([ 8.2744023 ,  6.85296131,  5.32454196,  3.62990861,  1.59240946,
        1.59240946,  3.62990861,  5.32454196,  6.85296131,  8.2744023 ])
array([-4.87947524, -5.29925691, -4.53529364, -2.66696586, -0.36007938,
       -0.47270639, -0.49809179, -0.42950048, -0.35893197, -0.29833225])
array([ 4.87947524,  5.29925691,  4.53529364,  2.66696586,  0.36007938,
        0.47270639,  0.49809179,  0.42950048,  0.35893197,  0.29833225])
array([-4.24227668, -4.52683182, -4.01645497, -2.66394847, -0.61658766,
       -0.47270639, -0.49809179, -0.42950048, -0.35893197, -0.29833225])
array([ 6.41771861,  6.83162332,  4.9203404 ,  2.34920927,  0.18597062,
        0.44674405,  0.48467656,  0.42254647,  0.35511487,  0.29613348])

Test derivative against numerical derivatives.

>>> np.vectorize(J_F)(y_squared, derivative=1, method="bessel") - np.vectorize(J_F)(y_squared, derivative=1, method="approx")
array([ -5.06825245e-03,   8.86800478e-05,   2.29852685e-05,
        -3.26269292e-04,  -2.71182328e-04,  -1.72205933e-07,
        -8.12912029e-08,   1.40641963e-07,  -4.34322898e-08,
         2.84767146e-08])

>>> np.vectorize(J_F)(y_squared, derivative=2, method="bessel") - np.vectorize(J_F)(y_squared, derivative=2, method="approx")
    array([ -5.28890689e-01,   4.34476713e-04,   2.28543869e-02,
             8.60189656e-03,   3.08677351e-02,   3.05994751e-02,
             1.99819211e-03,   4.25091227e-04,   1.31940097e-04,
             4.99716208e-05])

>>> np.vectorize(J_B)(y_squared, derivative=1, method="bessel") - np.vectorize(J_B)(y_squared, derivative=1, method="approx")
array([ -2.16951985e-04,  -2.55114955e-04,   6.97032234e-05,
        -3.38789396e-04,  -2.70543819e-04,   3.85995679e-07,
         1.32647813e-08,   1.30533110e-07,   1.23921645e-07,
        -3.70553466e-08])

>>> np.vectorize(J_B)(y_squared, derivative=2, method="bessel") - np.vectorize(J_B)(y_squared, derivative=2, method="approx")
array([ -1.18211875e-02,  -1.07143055e-02,  -2.78539577e-03,
         1.84933021e-02,   1.84354041e-01,   5.28891196e-02,
         2.65086492e-03,   5.05682480e-04,   1.48106517e-04,
         5.41656245e-05])
"""

# Load library

import _thermal_funcs

METHODS = ['quad', 'bessel', 'taylor', 'lim', 'approx', 'zeta']
METHODS_DERIV = ['bessel', 'approx']

def J_F(y_squared, method="bessel", derivative=0, **kwargs):
    """
    :param y_squared: Thermal function argument
    :type y_squared: float
    :param method: Method of calculating thermal function
    :type method: str
    :param derivative: Order of deriviatve
    :type derivative: int

    :returns: Thermal function J_F(y^2)
    :rtype: float
    """
    if derivative == 0:
        assert method in METHODS, "{} is not a known method: {}".format(method, METHODS)
        name = "J_F_{}".format(method)
    elif derivative == 1:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D1_J_F_{}".format(method)
    elif derivative == 2:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D2_J_F_{}".format(method)
    else:
        raise RuntimeError("Order must be 0 <= derivative <= 2")

    func = getattr(_thermal_funcs, name)
    return func(y_squared, **kwargs)

def J_B(y_squared, method="bessel", derivative=0, **kwargs):
    """
    :param y_squared: Thermal function argument
    :type y_squared: float
    :param method: Method of calculating thermal function
    :type method: str
    :param derivative: Order of deriviatve
    :type derivative: int

    :returns: Thermal function J_B(y^2)
    :rtype: float
    """
    if derivative == 0:
        assert method in METHODS, "{} is not a known method: {}".format(method, METHODS)
        name = "J_B_{}".format(method)
    elif derivative == 1:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D1_J_B_{}".format(method)
    elif derivative == 2:
        assert method in METHODS_DERIV, "{} is not a known method: {}".format(method, METHODS_DERIV)
        name = "D2_J_B_{}".format(method)
    else:
        raise RuntimeError("Order must be 0 <= derivative <= 2")

    func = getattr(_thermal_funcs, name)
    return func(y_squared, **kwargs)

if __name__ == '__main__':
    import doctest
    doctest.testmod(optionflags=doctest.REPORT_NDIFF)
    doctest.testmod()

<|endoftext|>


<commit_before>from __future__ import division
from percept.tasks.base import Task
from percept.tasks.train import Train
from percept.fields.base import Complex, List, Dict, Float
from inputs.inputs import SimpsonsFormats
from percept.utils.models import RegistryCategories, get_namespace
import logging
from percept.tests.framework import Tester
from percept.conf.base import settings
import re
import os
from matplotlib import pyplot
import numpy as np
from scikits.audiolab import Sndfile
from scikits.audiolab import oggread
import pandas as pd

log = logging.getLogger(__name__)

class LoadAudioFiles(Task):
    data = Complex()
    all_files = List()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    args = {
        'audio_dir' : os.path.abspath(os.path.join(settings.AUDIO_BASE_PATH, "audio")),
    }

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data, **kwargs)

    def extract_season(self,name):
        match1 = re.search('\[(\d+)[x\.](\d+)\]',name)
        if match1 is not None:
            season = match1.group(1)
            episode = match1.group(2)
            return int(season),int(episode)

        match2 = re.search('S(\d+)E(\d+)',name)
        if match2 is not None:
            season = match2.group(1)
            episode = match2.group(2)
            return int(season),int(episode)

        return None, None

    def calc_slope(self,x,y):
        x_mean = sum(x)/len(x)
        y_mean = sum(y)/len(y)
        x_dev = sum([abs(i-x_mean) for i in x])
        y_dev = sum([abs(i-y_mean) for i in y])

        slope = (x_dev*y_dev)/(x_dev*x_dev)
        return slope

    def get_indicators(self,vec):
        mean = sum(vec)
        slope = self.calc_slope(list(range(len(vec))),vec)
        std = np.std(vec)
        return mean, slope, std

    def calc_u(self,vec):
        fft = np.fft.fft(vec)
        return np.sum(np.multiply(fft,vec))/np.sum(vec)

    def calc_features(self, vec,freq):
        #bin count
        bc = 10
        bincount = list(range(bc))
        #framesize
        fsize = 512
        #mean
        m = np.mean(vec)
        #spectral flux
        sf = np.mean(vec-np.roll(vec,fsize))
        mx = np.max(vec)
        mi = np.min(vec)
        sdev = np.std(vec)
        binwidth = len(vec)/bc
        bins = []
        for i in xrange(0,bc):
            bins.append(vec[(i*binwidth):(binwidth*i + binwidth)])
        peaks = [np.max(i) for i in bins]
        mins = [np.min(i) for i in bins]
        amin,smin,stmin = self.get_indicators(mins)
        apeak, speak, stpeak = self.get_indicators(peaks)
        cepstrums = [np.fft.ifft(np.log(np.abs(np.fft.fft(i)))) for i in bins]
        inter = [self.get_indicators(i) for i in cepstrums]
        acep,scep, stcep = self.get_indicators([i[0] for i in inter])
        aacep,sscep, stsscep = self.get_indicators([i[1] for i in inter])

        zero_crossings = np.where(np.diff(np.sign(vec)))[0]
        zcc = len(zero_crossings)
        zccn = zcc/freq

        u = [self.calc_u(i) for i in bins]
        spread = np.sqrt(u[-1] - u[0]**2)
        skewness = (u[0]**3 - 3*u[0]*u[5] + u[-1])/spread**3

        #Spectral slope
        ss = self.calc_slope(range(len(vec)),np.fft.fft(vec))
        avss = self.calc_slope(bincount,[self.calc_slope(range(len(i)),np.fft.fft(i)) for i in bins])

        return [m,sf,mx,mi,sdev,amin,smin,stmin,apeak,speak,stpeak,acep,scep,stcep,aacep,sscep,stsscep,zcc,zccn,spread,skewness,ss,avss]

    def extract_features(self,sample,freq):
        left = self.calc_features(sample[:,0],freq)
        right = self.calc_features(sample[:,1],freq)
        return left+right

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        audio_dir = kwargs['audio_dir']
        all_files = []
        for ad in os.listdir(audio_dir):
            ad_path = os.path.abspath(os.path.join(audio_dir,ad))
            if os.path.isdir(ad_path):
                files = os.listdir(ad_path)
                all_files += [os.path.abspath(os.path.join(ad_path,f)) for f in files]
            else:
                all_files += [ad_path]
        self.all_files = [f for f in all_files if f.endswith(".ogg")]
        frames = []
        counter = 0
        for f in self.all_files:
            season,episode = self.extract_season(f)
            if season is None:
                continue
            subtitle_frame = data[((data['season']==season) & (data['episode']==episode))]
            if subtitle_frame.shape[0]==0:
                continue
            print "On file {0}".format(counter)
            counter+=1
            if counter>1:
                break
            f_data, fs, enc  = oggread(f)
            current_frame = 0
            subtitle_frame = subtitle_frame.sort('start')
            subtitle_frame.index = range(subtitle_frame.shape[0])
            audio_samples = []
            audio_features = []
            good_rows = []
            for i in xrange(0,subtitle_frame.shape[0]):
                start = subtitle_frame['start'].iloc[i]
                end = subtitle_frame['end'].iloc[i]
                samp = f_data[(start*fs):(end*fs),:]
                try:
                    features = self.extract_features(samp,fs)
                    audio_features.append(features)
                    good_rows.append(i)
                except Exception:
                    log.exception("Cannot generate features")
                    continue
            df = pd.concat([subtitle_frame.iloc[good_rows],pd.DataFrame(audio_features)],axis=1)
            df = df.fillna(-1)
            df.index = range(df.shape[0])
            frames.append(df)
        data = pd.concat(frames,axis=0)
        data.index = range(data.shape[0])

        return data<commit_msg>Use multiprocessing<commit_after>from __future__ import division
from percept.tasks.base import Task
from percept.tasks.train import Train
from percept.fields.base import Complex, List, Dict, Float
from inputs.inputs import SimpsonsFormats
from percept.utils.models import RegistryCategories, get_namespace
import logging
from percept.tests.framework import Tester
from percept.conf.base import settings
import re
import os
from matplotlib import pyplot
import numpy as np
from scikits.audiolab import Sndfile
from scikits.audiolab import oggread
import pandas as pd
from multiprocessing import Pool

log = logging.getLogger(__name__)

class LoadAudioFiles(Task):
    data = Complex()
    all_files = List()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    args = {
        'audio_dir' : os.path.abspath(os.path.join(settings.AUDIO_BASE_PATH, "audio")),
    }

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data, **kwargs)

    def extract_season(self,name):
        match1 = re.search('\[(\d+)[x\.](\d+)\]',name)
        if match1 is not None:
            season = match1.group(1)
            episode = match1.group(2)
            return int(season),int(episode)

        match2 = re.search('S(\d+)E(\d+)',name)
        if match2 is not None:
            season = match2.group(1)
            episode = match2.group(2)
            return int(season),int(episode)

        return None, None

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        audio_dir = kwargs['audio_dir']
        all_files = []
        for ad in os.listdir(audio_dir):
            ad_path = os.path.abspath(os.path.join(audio_dir,ad))
            if os.path.isdir(ad_path):
                files = os.listdir(ad_path)
                all_files += [os.path.abspath(os.path.join(ad_path,f)) for f in files]
            else:
                all_files += [ad_path]
        self.all_files = [f for f in all_files if f.endswith(".ogg")]
        frames = []
        counter = 0
        for f in self.all_files:
            season,episode = self.extract_season(f)
            if season is None:
                continue
            subtitle_frame = data[((data['season']==season) & (data['episode']==episode))]
            if subtitle_frame.shape[0]==0:
                continue
            print "On file {0}".format(counter)
            f_data, fs, enc  = oggread(f)
            subtitle_frame = subtitle_frame.sort('start')
            subtitle_frame.index = range(subtitle_frame.shape[0])
            samps = []
            for i in xrange(0,subtitle_frame.shape[0]):
                start = subtitle_frame['start'].iloc[i]
                end = subtitle_frame['end'].iloc[i]
                samp = f_data[(start*fs):(end*fs),:]
                samps.append({'samp' : samp, 'fs' : fs})
            p = Pool(4)
            results = p.map(process_subtitle, samps)
            good_rows = [i for i in xrange(0,len(results)) if results[i]!=None]
            audio_features = [i for i in results if i!=None]
            df = pd.concat([subtitle_frame.iloc[good_rows],pd.DataFrame(audio_features)],axis=1)
            df = df.fillna(-1)
            df.index = range(df.shape[0])
            frames.append(df)
        data = pd.concat(frames,axis=0)
        data.index = range(data.shape[0])

        return data

def calc_slope(x,y):
    x_mean = sum(x)/len(x)
    y_mean = sum(y)/len(y)
    x_dev = sum([abs(i-x_mean) for i in x])
    y_dev = sum([abs(i-y_mean) for i in y])

    slope = (x_dev*y_dev)/(x_dev*x_dev)
    return slope

def get_indicators(vec):
    mean = sum(vec)
    slope = calc_slope(list(range(len(vec))),vec)
    std = np.std(vec)
    return mean, slope, std

def calc_u(vec):
    fft = np.fft.fft(vec)
    return np.sum(np.multiply(fft,vec))/np.sum(vec)

def calc_features(vec,freq):
    #bin count
    bc = 10
    bincount = list(range(bc))
    #framesize
    fsize = 512
    #mean
    m = np.mean(vec)
    #spectral flux
    sf = np.mean(vec-np.roll(vec,fsize))
    mx = np.max(vec)
    mi = np.min(vec)
    sdev = np.std(vec)
    binwidth = len(vec)/bc
    bins = []
    for i in xrange(0,bc):
        bins.append(vec[(i*binwidth):(binwidth*i + binwidth)])
    peaks = [np.max(i) for i in bins]
    mins = [np.min(i) for i in bins]
    amin,smin,stmin = get_indicators(mins)
    apeak, speak, stpeak = get_indicators(peaks)
    cepstrums = [np.fft.ifft(np.log(np.abs(np.fft.fft(i)))) for i in bins]
    inter = [get_indicators(i) for i in cepstrums]
    acep,scep, stcep = get_indicators([i[0] for i in inter])
    aacep,sscep, stsscep = get_indicators([i[1] for i in inter])

    zero_crossings = np.where(np.diff(np.sign(vec)))[0]
    zcc = len(zero_crossings)
    zccn = zcc/freq

    u = [calc_u(i) for i in bins]
    spread = np.sqrt(u[-1] - u[0]**2)
    skewness = (u[0]**3 - 3*u[0]*u[5] + u[-1])/spread**3

    return [m,sf,mx,mi,sdev,amin,smin,stmin,apeak,speak,stpeak,acep,scep,stcep,aacep,sscep,stsscep,zcc,zccn,spread,skewness]

def extract_features(sample,freq):
    left = calc_features(sample[:,0],freq)
    right = calc_features(sample[:,1],freq)
    return left+right

def process_subtitle(d):
    samp = d['samp']
    fs = d['fs']
    try:
        features = extract_features(samp,fs)
    except Exception:
        log.exception("Cannot generate features")
        return None

    return features<|endoftext|>


<commit_before># -*- coding: utf8 -*-

# References:
# -----------
# https://tools.ietf.org/html/rfc791 - Internet Protocol
# https://tools.ietf.org/html/rfc792 - Internet Control Message Protocol (ICMP)

import random
import socket
import struct
import sys
import time
from io import BytesIO
from ctypes import BigEndianStructure, c_ubyte, c_uint16, c_uint32

PY3 = sys.version_info[0] == 3
if PY3:
    from io import StringIO
else:
    from StringIO import StringIO

protocols = {
    1: 'ICMP',
}

icmp_types = {
    0: 'Echo (ping) reply',
    8: 'Echo (ping) request',
}

class IPv4Header(BigEndianStructure):
    """
    IPv4 header
    """
    _pack_ = 1
    _fields_ = [
        ('version', c_ubyte, 4),
        ('ihl', c_ubyte, 4),
        ('dscp', c_ubyte, 6),
        ('ecn', c_ubyte, 2),
        ('total_length', c_uint16),
        ('identification', c_uint16),
        ('flags', c_uint16, 3),
        ('fragment_offset', c_uint16, 13),
        ('ttl', c_ubyte),
        ('protocol', c_ubyte),
        ('header_checksum', c_uint16),
        ('source_ip', c_uint32),
        ('destination_ip', c_uint32),
    ]

    def __str__(self):
        longest_name = max([len(k[0]) for k in self._fields_])
        indent_fmt = '{{:>{}s}}: {{}}\n'.format(longest_name)
        indent_verbose_fmt = '{{:>{}s}}: {{}} ({{}})\n'.format(longest_name)
        out = ''
        for k in self._fields_:
            value = getattr(self, k[0])
            if k[0] in ('source_ip', 'destination_ip'):
                out += indent_fmt.format(k[0], int_to_ip(value))
            elif k[0] in ('identification', 'header_checksum'):
                out += indent_verbose_fmt.format(k[0], value, hex(value))
            elif k[0] in ('protocol'):
                out += indent_verbose_fmt.format(k[0], value, protocols.get(value, '?'))
            else:
                out += indent_fmt.format(k[0], getattr(self, k[0]))
        return out.rstrip()

class ICMPEchoHeader(BigEndianStructure):
    """
    ICMP Echo header
    """
    _pack_ = 1
    _fields_ = [
        ('type', c_ubyte),
        ('code', c_ubyte),
        ('checksum', c_uint16),
        ('identifier', c_uint16),
        ('sequence_number', c_uint16),
    ]

    def __str__(self):
        longest_name = max([len(k[0]) for k in self._fields_])
        indent_fmt = '{{:>{}s}}: {{}}\n'.format(longest_name)
        indent_verbose_fmt = '{{:>{}s}}: {{}} ({{}})\n'.format(longest_name)
        out = ''
        for k in self._fields_:
            value = getattr(self, k[0])
            if k[0] in ('identifier', 'checksum'):
                out += indent_verbose_fmt.format(k[0], value, hex(value))
            elif k[0] in ('type'):
                out += indent_verbose_fmt.format(k[0], value, icmp_types.get(value, '?'))
            else:
                out += indent_fmt.format(k[0], value)
        return out.rstrip()

def ip_to_int(ip):
    """Encode IP address string as integer"""
    return struct.unpack('!I', socket.inet_aton(ip))[0]

def int_to_ip(value):
    """Decode integer to IP address string"""
    return socket.inet_ntoa(struct.pack('!I', value))

def print_hex(data):
    """Print data in hex format"""
    out = ''
    for i, x in enumerate(data):
        if i and i % 16 == 0:
            out += '\n'
        out += '%02x ' % ord(x)
    print(out)

def icmp_checksum(message):
    """Compute ICMP message checksum"""
    res = 0
    for value in message:
        res += value
        res = (res & 0xffff) + (res >> 16)
    return ~res & 0xffff

def make_ping_request():
    """Make ping request"""
    type_value = 8  # echo (ping) request
    code_value = 0
    checksum_value = 0
    id_value = random.randint(0, 0xffff)
    seq_num_value = 1
    data = struct.pack('>BBHHI', type_value, code_value, checksum_value, id_value, seq_num_value)
    str_buffer = StringIO(data.decode('latin') if PY3 else data)
    # update checksum
    chunks_16bits = []
    while True:
        value = str_buffer.read(2)
        if not value:
            break
        chunks_16bits.append(struct.unpack('H', value.encode('latin') if PY3 else value)[0])
    checksum_value = icmp_checksum(chunks_16bits)
    lst_data = list(data)
    if lst_data:
        lst_data[2:4] = struct.pack('H', checksum_value)
    return bytes(lst_data) if PY3 else ''.join(lst_data)

def ping(addr):
    """Send echo request (ping) to addr and get reply"""
    dest = ''
    try:
        dest = socket.gethostbyname(addr)
    except socket.gaierror:
        print('unknown host: {}'.format(addr))
        return

    print('PING {} ({})'.format(addr, dest))
    icmp_header = make_ping_request()

    # prepare payload
    icmp_header_bytes = BytesIO()
    icmp_header_bytes.write(icmp_header)
    icmp_header_bytes.seek(0)

    full_payload = icmp_header_bytes.read()
    print('request: {}'.format(full_payload.hex() if PY3 else full_payload.encode('hex')))

    response = ''
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.getprotobyname('icmp'))
        s.settimeout(1)
        start_time = time.time()
        s.sendto(full_payload, (dest, 0))
        response, _ = s.recvfrom(256)
        end_time = time.time()
    except Exception as ex:
        print(ex)
        return
    if not response:
        return
    print('response: {}'.format(response.hex() if PY3 else response.encode('hex')))
    print('time = {:.2f} ms'.format((end_time - start_time) * 1000))

    # read IPv4 header
    response_bytes = BytesIO(response)
    ipv4_header = IPv4Header()
    response_bytes.readinto(ipv4_header)

    # read ICMP header
    icmp_echo_header = ICMPEchoHeader()
    response_bytes.readinto(icmp_echo_header)

    print('=====================================================')
    print('IPv4 header')
    print('=====================================================')
    print(ipv4_header)

    print('=====================================================')
    print('ICMP Echo header')
    print('=====================================================')
    print(icmp_echo_header)

def main():
    address = 'www.google.com'
    ping(address)

if __name__ == '__main__':
    main()
<commit_msg>Update icmp.py<commit_after># -*- coding: utf8 -*-

# References:
# -----------
# https://tools.ietf.org/html/rfc791 - Internet Protocol
# https://tools.ietf.org/html/rfc792 - Internet Control Message Protocol (ICMP)

import random
import socket
import struct
import sys
import time
from io import BytesIO
from ctypes import BigEndianStructure, c_ubyte, c_uint16, c_uint32

PY3 = sys.version_info[0] == 3
if PY3:
    from io import StringIO
else:
    from StringIO import StringIO

protocols = {
    1: 'ICMP',
}

icmp_types = {
    0: 'Echo (ping) reply',
    8: 'Echo (ping) request',
}

class IPv4Header(BigEndianStructure):
    """
    IPv4 header
    """
    _pack_ = 1
    _fields_ = [
        ('version', c_ubyte, 4),
        ('ihl', c_ubyte, 4),
        ('dscp', c_ubyte, 6),
        ('ecn', c_ubyte, 2),
        ('total_length', c_uint16),
        ('identification', c_uint16),
        ('flags', c_uint16, 3),
        ('fragment_offset', c_uint16, 13),
        ('ttl', c_ubyte),
        ('protocol', c_ubyte),
        ('header_checksum', c_uint16),
        ('source_ip', c_uint32),
        ('destination_ip', c_uint32),
    ]

    def __str__(self):
        longest_name = max([len(k[0]) for k in self._fields_])
        indent_fmt = '{{:>{}s}}: {{}}\n'.format(longest_name)
        indent_verbose_fmt = '{{:>{}s}}: {{}} ({{}})\n'.format(longest_name)
        out = ''
        for k in self._fields_:
            value = getattr(self, k[0])
            if k[0] in ('source_ip', 'destination_ip'):
                out += indent_fmt.format(k[0], int_to_ip(value))
            elif k[0] in ('identification', 'header_checksum'):
                out += indent_verbose_fmt.format(k[0], value, hex(value))
            elif k[0] in ('protocol'):
                out += indent_verbose_fmt.format(k[0], value, protocols.get(value, '?'))
            else:
                out += indent_fmt.format(k[0], getattr(self, k[0]))
        return out.rstrip()

class ICMPEchoHeader(BigEndianStructure):
    """
    ICMP Echo header
    """
    _pack_ = 1
    _fields_ = [
        ('type', c_ubyte),
        ('code', c_ubyte),
        ('checksum', c_uint16),
        ('identifier', c_uint16),
        ('sequence_number', c_uint16),
    ]

    def __str__(self):
        longest_name = max([len(k[0]) for k in self._fields_])
        indent_fmt = '{{:>{}s}}: {{}}\n'.format(longest_name)
        indent_verbose_fmt = '{{:>{}s}}: {{}} ({{}})\n'.format(longest_name)
        out = ''
        for k in self._fields_:
            value = getattr(self, k[0])
            if k[0] in ('identifier', 'checksum'):
                out += indent_verbose_fmt.format(k[0], value, hex(value))
            elif k[0] in ('type'):
                out += indent_verbose_fmt.format(k[0], value, icmp_types.get(value, '?'))
            else:
                out += indent_fmt.format(k[0], value)
        return out.rstrip()

def ip_to_int(ip):
    """Encode IP address string as integer"""
    return struct.unpack('!I', socket.inet_aton(ip))[0]

def int_to_ip(value):
    """Decode integer to IP address string"""
    return socket.inet_ntoa(struct.pack('!I', value))

def print_hex(data):
    """Print data in hex format"""
    out = ''
    for i, x in enumerate(data):
        if i and i % 16 == 0:
            out += '\n'
        out += '%02x ' % ord(x)
    print(out)

def icmp_checksum(message):
    """Compute ICMP message checksum"""
    res = 0
    for value in message:
        res += value
        res = (res & 0xffff) + (res >> 16)
    return ~res & 0xffff

def make_ping_request():
    """Make ping request"""
    type_value = 8  # echo (ping) request
    code_value = 0
    checksum_value = 0
    id_value = random.randint(0, 0xffff)
    seq_num_value = 1
    data = struct.pack('>BBHHI', type_value, code_value, checksum_value, id_value, seq_num_value)
    str_buffer = StringIO(data.decode('latin'))
    # update checksum
    chunks_16bits = []
    while True:
        value = str_buffer.read(2)
        if not value:
            break
        chunks_16bits.append(struct.unpack('H', value.encode('latin'))[0])
    checksum_value = icmp_checksum(chunks_16bits)
    lst_data = list(data)
    if lst_data:
        lst_data[2:4] = struct.pack('H', checksum_value)
    return bytes(lst_data) if PY3 else ''.join(lst_data)

def ping(addr):
    """Send echo request (ping) to addr and get reply"""
    dest = ''
    try:
        dest = socket.gethostbyname(addr)
    except socket.gaierror:
        print('unknown host: {}'.format(addr))
        return

    print('PING {} ({})'.format(addr, dest))
    icmp_header = make_ping_request()

    # prepare payload
    icmp_header_bytes = BytesIO()
    icmp_header_bytes.write(icmp_header)
    icmp_header_bytes.seek(0)

    full_payload = icmp_header_bytes.read()
    print('request: {}'.format(full_payload.hex() if PY3 else full_payload.encode('hex')))

    response = ''
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.getprotobyname('icmp'))
        s.settimeout(1)
        start_time = time.time()
        s.sendto(full_payload, (dest, 0))
        response, _ = s.recvfrom(256)
        end_time = time.time()
    except Exception as ex:
        print(ex)
        return
    if not response:
        return
    print('response: {}'.format(response.hex() if PY3 else response.encode('hex')))
    print('time = {:.2f} ms'.format((end_time - start_time) * 1000))

    # read IPv4 header
    response_bytes = BytesIO(response)
    ipv4_header = IPv4Header()
    response_bytes.readinto(ipv4_header)

    # read ICMP header
    icmp_echo_header = ICMPEchoHeader()
    response_bytes.readinto(icmp_echo_header)

    print('=====================================================')
    print('IPv4 header')
    print('=====================================================')
    print(ipv4_header)

    print('=====================================================')
    print('ICMP Echo header')
    print('=====================================================')
    print(icmp_echo_header)

def main():
    address = 'www.google.com'
    ping(address)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "d9e46beace3120fbc4810dda5c3ed88f93e862a4"
    LLVM_SHA256 = "65f613519dbce76c768c0ee636d1a414c4c9e4e12806d9b6bf2d3af012f0714d"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-{commit}".format(commit = LLVM_COMMIT),
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        build_file = "//third_party/llvm:BUILD.bazel",
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@ed2f0ad30719<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "ed2f0ad307193165a64c9847c24d8c9f4219f42b"
    LLVM_SHA256 = "8067a94bb4e8f58466b5d7bb03faf99701d0c6bf876d9abaca0252e1b02fc499"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-{commit}".format(commit = LLVM_COMMIT),
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        build_file = "//third_party/llvm:BUILD.bazel",
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
<commit_msg>Almost partially functional!<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
from scipy.optimize import fmin_ncg

class NeuralNetwork:
    """A simple neural network."""

    def __init__(self, hidden_layers=(25,), reg_lambda=0, num_labels=2):
        """Instantiates the class."""
        self.__hidden_layers = tuple(hidden_layers)
        self.__lambda = reg_lambda
        if num_labels > 2:
            self.__num_labels = num_labels
        else:
            self.__num_labels = 1

    def train(self, training_set, iterations=500):
        """Trains itself using the sequence data."""
        self.__X = np.matrix([example[0] for example in training_set])
        if self.__num_labels == 1:
            self.__y = np.matrix([example[1] for example in training_set]).reshape((-1, 1))
        else:
            eye = np.eye(self.__num_labels)
            self.__y = np.matrix([eye[example[1]] for example in training_set])
        self.__m = self.__X[0].shape[0]
        self.__input_layer_size = self.__X[0].shape[1]
        self.__sizes = [self.__input_layer_size]
        self.__sizes.extend(self.__hidden_layers)
        self.__sizes.append(self.__num_labels)
        param_size = 0
        for count in range(len(self.__sizes) - 1):
            param_size += (self.__sizes[count] + 1) * self.__sizes[count+1]
        print(param_size)
        # self.__thetas = self.__roll(fmin_ncg(self.__cost, np.zeros(param_size), self.__costGrad))

    def __cost(self, params):
        """Computes cost function."""

    def __costGrad(self, params):
        """Computes cost function derivative."""

    def __roll(self, unrolled):
        """Converts parameter array back into matrices."""
        rolled = []
        index = 0
        for count in range(len(self.__sizes) - 1):
            in_size = self.__sizes[count]
            out_size = self.__sizes[count+1]
            theta_unrolled = np.matrix(unrolled[index:index+(in_size+1)*out_size])
            theta_rolled = theta_unrolled.reshape((out_size, in_size+1))
            rolled.append(theta_rolled)
            index += (in_size + 1) * out_size
        return rolled

    def __unroll(self, rolled):
        """Converts parameter matrices into an array."""
        return np.array(np.concatenate([matrix.reshape(-1) for matrix in rolled], axis=1)).reshape(-1)
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "f4d78a5e3aee53d46e0f13e77f08ee610bade7fc"
    LLVM_SHA256 = "2514974c699945620281a99be2b4c3c26e48a30c9213d749275a3f2601967400"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@7f6e3316456f<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "7f6e3316456f939a062aad0eeaac983251a1747c"
    LLVM_SHA256 = "83d80a34a4ee6672d10979044d6fba5ea0383f8565c371383a0b5d321605d326"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "6359842bc088857799318dad366099eeca92a4d5"
    LLVM_SHA256 = "3c349ceaeae29d8b473b2bbeb91966d71fd467672e68b6156b9a44e28f61658a"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@f3c577ed38e5<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "f3c577ed38e55dca46692313f5b76688a115861a"
    LLVM_SHA256 = "b6f517337b0327637755ed08851cb0a0ff924d08b65983789033a28b661e083f"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "c0ef93bec85a8847b51d91d2a6470af903e1ec9a"
    LLVM_SHA256 = "5c8b91d77319abd2c6fde30f89cd52b018a11ce0f4173b37911592d1ab97d05c"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@628dda08b82f<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tf_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "628dda08b82fcedcd9e89c9ef7850388e988bf68"
    LLVM_SHA256 = "b48c8c63c17631cc0160e1359c1e977188aa6cf5924cfd4b5664397effe65f30"

    tf_http_archive(
        name = name,
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
        link_files = {
            "//third_party/llvm:llvm.autogenerated.BUILD": "llvm/BUILD",
            "//third_party/mlir:BUILD": "mlir/BUILD",
            "//third_party/mlir:test.BUILD": "mlir/test/BUILD",
        },
    )
<|endoftext|>


<commit_before>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tfrt_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "d384f2b253bf5150eda9e43ce4827f7b3f33da20"
    LLVM_SHA256 = "a4401f3b84e317e8c2a8f11892bb51509fdd9f365aeaf5edd302ffa4fc4efe4d"

    tfrt_http_archive(
        name = name,
        build_file = "//third_party/llvm:BUILD",
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
    )
<commit_msg>Integrate LLVM at llvm/llvm-project@69f7210657a3<commit_after>"""Provides the repository macro to import LLVM."""

load("//third_party:repo.bzl", "tfrt_http_archive")

def repo(name):
    """Imports LLVM."""
    LLVM_COMMIT = "69f7210657a302d0f0a756eb91ec4f54d85fb704"
    LLVM_SHA256 = "a4d3df66d466825cb9992b3a06468c0330ae05f07c6c0f7b703624cbd32a5859"

    tfrt_http_archive(
        name = name,
        build_file = "//third_party/llvm:BUILD",
        sha256 = LLVM_SHA256,
        strip_prefix = "llvm-project-" + LLVM_COMMIT,
        urls = [
            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
            "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT),
        ],
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division

import numpy as np
import numpy.linalg as nl

import logging

def jacobian(F,x,h=1e-6):
    """
    Numerical Jacobian at x.
    """
    x = np.array(x)
    L = x.size
    vhs = h * np.identity(L)
    Fs = np.array([F(x+vh) for vh in vhs])
    grad = (Fs - F(x))/h
    return np.array(grad).T

class RootSolver(object):
    def __init__(self, F=None, level=0.):
        self.F = F
        self.level = level

    def residual(self, x):
        a = x.reshape(self.shape)
        res = self.F(a)
        try:
            res_vec = res.ravel()
        except AttributeError: # a list of arrays
            res_vec = np.hstack([comp.ravel() for comp in res])
        return res_vec

    def get_initial(self, x0):
        if np.isscalar(x0):
            x = np.array([x0])
        else:
            x = np.array(x0)
        self.shape = x.shape
        x = x.ravel()
        return x

    def get_result(self, x):
        return x.reshape(self.shape)

    class DidNotConverge(Exception):
        """
        Exception raised when the non linear solver does not converge.
        """

class Newton(RootSolver):
    """
    Simple Newton solver to solve F(x) = level.
    """

    h = 1e-6
    def der(self, x):
        return jacobian(self.residual, x, self.h)

    maxiter = 600
    tol = 1e-11
    def run(self, x0):
        """
        Run the Newton iteration.
        """
        x = self.get_initial(x0)
        for i in range(self.maxiter):
            d = self.der(x)
            y = self.level - self.residual(x)
            if np.isscalar(y):
                incr = y/d.item()
            else:
                try:
                    incr = np.linalg.solve(d, y)
                except np.linalg.LinAlgError as ex:
                    eigvals, eigs = np.linalg.eig(d)
                    zerovecs = eigs[:, np.abs(eigvals) < 1e-10]
                    raise np.linalg.LinAlgError("%s: %s %s" % (ex.message, repr(zerovecs), eigvals))
##                  raise np.linalg.LinAlgError("Condition: %.1e" % np.linalg.cond(d))
            x += incr
            if nl.norm(incr) < self.tol:
                break
##          if self.is_zero(x):
##              break
        else:
            raise self.DidNotConverge(u"Newton algorithm did not converge after %d iterations. Dx=%.2e" % (i, nl.norm(incr)))
        self.required_iter = i
        return self.get_result(x)

    def is_zero(self, x): # use np.allclose?
        res = nl.norm(self.F(x) - self.level)
        return res < self.tol

class FSolve(RootSolver):
    """
    Wrapper around scipy.optimize.fsolve
    """
    xtol = 1.49012e-08 # default fsolve xtol
    def run(self, x0):
        guess = self.get_initial(x0)
        import scipy.optimize
        full_output = scipy.optimize.fsolve(self.residual, guess, full_output=True,  xtol = self.xtol, )
        result, self.info, success, msg = full_output
        if success != 1:
            raise self.DidNotConverge(msg)
        return self.get_result(result)

class MultipleSolver(object):
    solver_classes = [FSolve, Newton]
    def __init__(self, residual=None, level=0):
        self.solvers = [S(residual,level) for S in self.solver_classes]

    def run(self, guess):
        for solver in self.solvers:
            try:
                root = solver.run(guess)
            except solver.DidNotConverge as e:
                last_exception = e
                logging.info("Switch nonlinear solver")
            else:
                return root
        else:
            raise last_exception

<commit_msg>remove old stop check<commit_after># -*- coding: utf-8 -*-
from __future__ import division

import numpy as np
import numpy.linalg as nl

import logging

def jacobian(F,x,h=1e-6):
    """
    Numerical Jacobian at x.
    """
    x = np.array(x)
    L = x.size
    vhs = h * np.identity(L)
    Fs = np.array([F(x+vh) for vh in vhs])
    grad = (Fs - F(x))/h
    return np.array(grad).T

class RootSolver(object):
    def __init__(self, F=None, level=0.):
        self.F = F
        self.level = level

    def residual(self, x):
        a = x.reshape(self.shape)
        res = self.F(a)
        try:
            res_vec = res.ravel()
        except AttributeError: # a list of arrays
            res_vec = np.hstack([comp.ravel() for comp in res])
        return res_vec

    def get_initial(self, x0):
        if np.isscalar(x0):
            x = np.array([x0])
        else:
            x = np.array(x0)
        self.shape = x.shape
        x = x.ravel()
        return x

    def get_result(self, x):
        return x.reshape(self.shape)

    class DidNotConverge(Exception):
        """
        Exception raised when the non linear solver does not converge.
        """

class Newton(RootSolver):
    """
    Simple Newton solver to solve F(x) = level.
    """

    h = 1e-6
    def der(self, x):
        return jacobian(self.residual, x, self.h)

    maxiter = 600
    tol = 1e-11
    def run(self, x0):
        """
        Run the Newton iteration.
        """
        x = self.get_initial(x0)
        for i in range(self.maxiter):
            d = self.der(x)
            y = self.level - self.residual(x)
            if np.isscalar(y):
                incr = y/d.item()
            else:
                try:
                    incr = np.linalg.solve(d, y)
                except np.linalg.LinAlgError as ex:
                    eigvals, eigs = np.linalg.eig(d)
                    zerovecs = eigs[:, np.abs(eigvals) < 1e-10]
                    raise np.linalg.LinAlgError("%s: %s %s" % (ex.message, repr(zerovecs), eigvals))
##                  raise np.linalg.LinAlgError("Condition: %.1e" % np.linalg.cond(d))
            x += incr
            if nl.norm(incr) < self.tol:
                break
        else:
            raise self.DidNotConverge(u"Newton algorithm did not converge after %d iterations. Dx=%.2e" % (i, nl.norm(incr)))
        self.required_iter = i
        return self.get_result(x)

    def is_zero(self, x): # use np.allclose?
        res = nl.norm(self.F(x) - self.level)
        return res < self.tol

class FSolve(RootSolver):
    """
    Wrapper around scipy.optimize.fsolve
    """
    xtol = 1.49012e-08 # default fsolve xtol
    def run(self, x0):
        guess = self.get_initial(x0)
        import scipy.optimize
        full_output = scipy.optimize.fsolve(self.residual, guess, full_output=True,  xtol = self.xtol, )
        result, self.info, success, msg = full_output
        if success != 1:
            raise self.DidNotConverge(msg)
        return self.get_result(result)

class MultipleSolver(object):
    solver_classes = [FSolve, Newton]
    def __init__(self, residual=None, level=0):
        self.solvers = [S(residual,level) for S in self.solver_classes]

    def run(self, guess):
        for solver in self.solvers:
            try:
                root = solver.run(guess)
            except solver.DidNotConverge as e:
                last_exception = e
                logging.info("Switch nonlinear solver")
            else:
                return root
        else:
            raise last_exception

<|endoftext|>


<commit_before>import os

from agx.core import (
    handler,
    Scope,
    registerScope,
    token
)
from node.ext.zcml import ZCMLNode
from node.ext.zcml import ZCMLFile
from node.ext.zcml import SimpleDirective
from node.ext.zcml import ComplexDirective
from node.ext.uml.utils import TaggedValues, UNSET

confname = 'configure.zcml'

def relpath(self,other):
    assert self.path == other.path[:len(self.path)], \
           'other must be a subpath of self'
    return '/'.join(other.path[len(self.path):])

def addZcmlRef(directory, zcml):
    """Adds a reference to a zcml file and implicitly taks care for creating
    a configure.zcml"""
    # zcmls zamsammeln, um zu wissen, was vom configure.zcml includiert
    # werden soll
    path = directory.path
    path.append(confname)
    #fullpath = os.path.join(*path)
    if confname not in directory.keys():
        #conf = ZCMLFile(fullpath)
        conf = ZCMLFile()
        directory[confname] = conf
    else:
        conf = directory[confname]

    zcmlpath = relpath(directory,zcml)
    if zcmlpath != 'configure.zcml':
        found_include = conf.filter(tag='include', attr='file', value=zcmlpath)
        #add include directive if necessary
        if not found_include:
            include = SimpleDirective(name='include', parent=conf)
            include.attrs['file'] = zcmlpath

    if directory not in token('pyeggs',False).directories:
        parentdir = directory.__parent__
        addZcmlRef(parentdir, conf)

def zcml_include_package(directory):
    if not 'configure.zcml' in directory.parent:
        directory.parent['configure.zcml'] = ZCMLFile()
    zcml = directory.parent['configure.zcml']
    package = '.%s' % directory.name
    if not zcml.filter(tag='include', attr='package', value=package):
        include = SimpleDirective(name='include', parent=zcml)
        include.attrs['package'] = package<commit_msg>sub-configures are now included via include package<commit_after>import os

from agx.core import (
    handler,
    Scope,
    registerScope,
    token
)
from node.ext.zcml import ZCMLNode
from node.ext.zcml import ZCMLFile
from node.ext.zcml import SimpleDirective
from node.ext.zcml import ComplexDirective
from node.ext.uml.utils import TaggedValues, UNSET

confname = 'configure.zcml'

def relpath(self,other):
    assert self.path == other.path[:len(self.path)], \
           'other must be a subpath of self'
    return '/'.join(other.path[len(self.path):])

def addZcmlRef(directory, zcml):
    """Adds a reference to a zcml file and implicitly taks care for creating
    a configure.zcml"""
    # zcmls zamsammeln, um zu wissen, was vom configure.zcml includiert
    # werden soll
    path = directory.path
    path.append(confname)
    #fullpath = os.path.join(*path)
    if confname not in directory.keys():
        #conf = ZCMLFile(fullpath)
        conf = ZCMLFile()
        directory[confname] = conf
    else:
        conf = directory[confname]

    zcmlpath = relpath(directory,zcml)
    if zcml.name!='configure.zcml':
        found_include = conf.filter(tag='include', attr='file', value=zcmlpath)
        #add include directive if necessary
        if not found_include:
            include = SimpleDirective(name='include', parent=conf)
            include.attrs['file'] = zcmlpath
    else:
        packname="."+directory.name
        found_include = conf.filter(tag='include', attr='package', value=packname)
        #add include directive if necessary
        if not found_include:
            include = SimpleDirective(name='include', parent=conf)
            include.attrs['package'] = packname

    if directory not in token('pyeggs',False).directories:
        parentdir = directory.__parent__
        addZcmlRef(parentdir, conf)

def zcml_include_package(directory):
    import pdb;pdb.set_trace()
    if not 'configure.zcml' in directory.parent:
        directory.parent['configure.zcml'] = ZCMLFile()
    zcml = directory.parent['configure.zcml']
    package = '.%s' % directory.name
    if not zcml.filter(tag='include', attr='package', value=package):
        include = SimpleDirective(name='include', parent=zcml)
        include.attrs['package'] = package<|endoftext|>