<commit_before>from collections import defaultdict
from os.path import basename
import fingerprinting
import read_audio

# threshold for number of matching fingerprints in a given time offset
# to produce a match
MATCH_THRESHOLD = 60

def get_matches_for_hashes(hashes, dstore):
    """ hashes are a list of (md5, offset, time)
    Returns a list of (song_id, offset_diff) for each match.
    """
    # gets dict of md5 -> (offset, song_id, time)
    stored_hashes = dstore.get_fingerprints()

    # get all tuples that match our hashes.
    # matches will contain a list of tuples of the form:
    # (md5, database_offset, database_song_id)
    matches = []
    for h in hashes:
        if h[0] in stored_hashes:
            # list of tuples (multiple database songs with same hash)
            tuples = stored_hashes.get(h[0])
            matches.extend([(h[0],) + t for t in tuples])

    # create a dictionary of our query song offsets from the hashes
    # {hash: offset}
    query_offsets = {t[0]: t[1:] for t in hashes}

    # returns a list of (song_id, offset_diff, offset, query_offset)

    # offset_diff is the difference between the offset of the
    # fingerprint in the sample we've been given and the offset of the
    # corresponding fingerprint in the file in the datastore.

    res = []
    for h, db_offset, db_song_id, db_time in matches:
        query_offset = query_offsets[h][0]
        query_time = query_offsets[h][1]
        offset_diff = db_offset - query_offset
        res.append((db_song_id, offset_diff, db_offset, query_time, db_time))
    return res

def get_match(hash_tuples, dstore):
    """hash_tuples is a list of (MD5, offset) -> list of (songid, offset_dif)
    Assumes that what you're matching against is already in the datastore.
    """
    # (db_song_id, offset_diff, db_offset, query_time, db_time)
    matches = get_matches_for_hashes(hash_tuples, dstore)
    # make a dict of {song_name: {offset_diff: #collisions}}
    # then, we can iterate through the dict and choose songs that
    # have more than MATCH_THRESHOLD matches for some offset.
    song_offset_counter = defaultdict(lambda: defaultdict(int))
    for m in matches:
        song_id, offset_diff = m[:2]
        song_offset_counter[song_id][offset_diff] += 1

    likely_matches = []  # where we'll store all the matches
    for song_id, diff_dict in song_offset_counter.iteritems():
        offset_diff, max_count = max(diff_dict.iteritems(), key=lambda t: t[1])
        if max_count < MATCH_THRESHOLD:
            continue
        # append it to our list of likely matches.
        # we could also add a measure of confidence (how many matches
        # above our threshold)
        likely_matches.append((song_id, offset_diff))

    matches_to_return = []
    for song_id, offset_diff in likely_matches:
        # pick out hashes for which we have the right song id and the
        # right offset difference
        hashes = [match for match in matches
                  if match[0] == song_id and match[1] == offset_diff]

        # find minimum by offset
        start_peak = min(hashes, key=lambda t: t[2])
        query_start_time = start_peak[3]
        db_start_time = start_peak[4]

        end_peak = max(hashes, key=lambda t: t[2])
        query_end_time = end_peak[3]
        db_end_time = end_peak[4]

        query_duration = query_end_time - query_start_time
        db_duration = db_end_time - db_start_time

        # return the most likely song's ID and the start and end times:
        song_name = dstore.get_song_file_from_id(song_id)

        #if query_duration > 5 and db_duration > 5:
        matches_to_return.append((song_name, query_start_time, db_start_time))
    return matches_to_return

def is_match(f1, f2):
    """Returns True if f1 matches to f2.

    Currently works by comparing the number of matched fingerprints
    that occur at the same offset difference between the two songs.

    If this number is greater than MATCH_THRESHOLD, then it returns
    True.

    Wow, code reuse. Remind yourself to kick dan.
    Not very :herb:

    TODO: WHY IS THIS SO SLOW

    """
    mono1, mono2 = read_audio.get_mono(f1), read_audio.get_mono(f2)
    prints1, prints2 = [dict(fingerprinting.get_fingerprints(m))
                        for m in (mono1, mono2)]
    shared_hashes = set(prints1.keys()).intersection(set(prints2.keys()))

    # {offset_diff -> [(md5, offset1, offset2), ...]}
    offset_dict = defaultdict(list)
    match_counter = defaultdict(int)

    max_count = 0
    offset_difference = 0

    for h in shared_hashes:
        offset1, offset2 = prints1[h], prints2[h]
        offset_diff = abs(offset1 - offset2)
        offset_dict[offset_diff].append((h, offset1, offset2))
        match_counter[offset_diff] += 1
        count = match_counter[offset_diff]
        if count > max_count:
            max_count = count
            offset_difference = offset_diff

    if max_count > MATCH_THRESHOLD:
        hashes = match_counter[offset_difference]
        start1 = min(hashes, key=lambda t: t[1])[1]
        start2 = min(hashes, key=lambda t: t[2])[2]

        return (start1, start2)

    else:
        return False

def print_match(audio_1_path, match_data):
    """Prints matches according to black-box spec
    """

    for match in match_data:
        audio_2_path = match[0]
        time_1 = match[1]
        time_2 = match[2]

        # python will automatically include spaces between comma-deliniated elements
        print "MATCH", basename(audio_1_path).lstrip(), basename(audio_2_path).lstrip(), time_1, time_2

# """
# is_match:
# Returns a boolean if we deem that f1 and f2 match

# INPUT: 2 files that are valid file paths
# OUTPUT: True if we deem the files match, otherwise False
# """
# def is_match(f1, f2):

#     #preapre a string for our output
#     match_threshold = 150000000000 # new threshold from new trial and error
#     match_coefficient = 0

#     if ( read_audio.is_mp3(f1) ):
#         f1 = read_audio.create_temp_wav_file(f1)

#     if ( read_audio.is_mp3(f2) ):
#         f2 = read_audio.create_temp_wav_file(f2)

#     """
#     validate that the two files are of the same length
#     """
#     """TODO: Remove after ASSN 5"""
#     if(read_audio.length(f1) != read_audio.length(f2)):
#         read_audio.delete_temp_file(f1) # only deletes if /tmp is in filepath
#         read_audio.delete_temp_file(f2)
#         return False
#     else:
#         # get our match coefficient!
#         match_coefficient = similarity(f1, f2)
#         read_audio.delete_temp_file(f1) # only deletes if /tmp is in filepath
#         read_audio.delete_temp_file(f2)

#     #final print out to SDTOUT
#     if(match_coefficient < match_threshold):
#         return True
#     else:
#         return False

# """
# match_files:
# Compares all files, prints matches

# INPUT: 2 file arrays
# OUTPUT: List of matches as tuples, also prints all matches
# """
# def match_files(a1, a2):
#     matches = []
#     for f1 in a1:
#         for f2 in a2:
#             result = is_match(f1,f2)
#             if (result):
#                 matches.append((f1,f2))
#                 final_print(f1,f2)
#     return matches

# """
# is_match:
# Returns a boolean if we deem that f1 and f2 match

# INPUT: 2 files that are valid file paths
# OUTPUT: True if we deem the files match, otherwise False
# """
# def is_match(f1, f2):

#     #preapre a string for our output
#     match_threshold = 150000000000 # new threshold from new trial and error
#     match_coefficient = 0

#     if ( read_audio.is_mp3(f1) ):
#         f1 = read_audio.create_temp_wav_file(f1)

#     if ( read_audio.is_mp3(f2) ):
#         f2 = read_audio.create_temp_wav_file(f2)

#     """
#     validate that the two files are of the same length
#     """
#     """TODO: Remove after ASSN 5"""
#     if(read_audio.length(f1) != read_audio.length(f2)):
#         read_audio.delete_temp_file(f1) # only deletes if /tmp is in filepath
#         read_audio.delete_temp_file(f2)
#         return False
#     else:
#         # get our match coefficient!
#         match_coefficient = similarity(f1, f2)
#         read_audio.delete_temp_file(f1) # only deletes if /tmp is in filepath
#         read_audio.delete_temp_file(f2)

#     #final print out to SDTOUT
#     if(match_coefficient < match_threshold):
#         return True
#     else:
#         return False
<commit_msg>Got rid of old is_match methods<commit_after>from collections import defaultdict
from os.path import basename
import fingerprinting
import read_audio

# threshold for number of matching fingerprints in a given time offset
# to produce a match
MATCH_THRESHOLD = 60

def get_matches_for_hashes(hashes, dstore):
    """ hashes are a list of (md5, offset, time)
    Returns a list of (song_id, offset_diff) for each match.
    """
    # gets dict of md5 -> (offset, song_id, time)
    stored_hashes = dstore.get_fingerprints()

    # get all tuples that match our hashes.
    # matches will contain a list of tuples of the form:
    # (md5, database_offset, database_song_id)
    matches = []
    for h in hashes:
        if h[0] in stored_hashes:
            # list of tuples (multiple database songs with same hash)
            tuples = stored_hashes.get(h[0])
            matches.extend([(h[0],) + t for t in tuples])

    # create a dictionary of our query song offsets from the hashes
    # {hash: offset}
    query_offsets = {t[0]: t[1:] for t in hashes}

    # returns a list of (song_id, offset_diff, offset, query_offset)

    # offset_diff is the difference between the offset of the
    # fingerprint in the sample we've been given and the offset of the
    # corresponding fingerprint in the file in the datastore.

    res = []
    for h, db_offset, db_song_id, db_time in matches:
        query_offset = query_offsets[h][0]
        query_time = query_offsets[h][1]
        offset_diff = db_offset - query_offset
        res.append((db_song_id, offset_diff, db_offset, query_time, db_time))
    return res

def get_match(hash_tuples, dstore):
    """hash_tuples is a list of (MD5, offset) -> list of (songid, offset_dif)
    Assumes that what you're matching against is already in the datastore.
    """
    # (db_song_id, offset_diff, db_offset, query_time, db_time)
    matches = get_matches_for_hashes(hash_tuples, dstore)
    # make a dict of {song_name: {offset_diff: #collisions}}
    # then, we can iterate through the dict and choose songs that
    # have more than MATCH_THRESHOLD matches for some offset.
    song_offset_counter = defaultdict(lambda: defaultdict(int))
    for m in matches:
        song_id, offset_diff = m[:2]
        song_offset_counter[song_id][offset_diff] += 1

    likely_matches = []  # where we'll store all the matches
    for song_id, diff_dict in song_offset_counter.iteritems():
        offset_diff, max_count = max(diff_dict.iteritems(), key=lambda t: t[1])
        if max_count < MATCH_THRESHOLD:
            continue
        # append it to our list of likely matches.
        # we could also add a measure of confidence (how many matches
        # above our threshold)
        likely_matches.append((song_id, offset_diff))

    matches_to_return = []
    for song_id, offset_diff in likely_matches:
        # pick out hashes for which we have the right song id and the
        # right offset difference
        hashes = [match for match in matches
                  if match[0] == song_id and match[1] == offset_diff]

        # find minimum by offset
        start_peak = min(hashes, key=lambda t: t[2])
        query_start_time = start_peak[3]
        db_start_time = start_peak[4]

        end_peak = max(hashes, key=lambda t: t[2])
        query_end_time = end_peak[3]
        db_end_time = end_peak[4]

        query_duration = query_end_time - query_start_time
        db_duration = db_end_time - db_start_time

        # return the most likely song's ID and the start and end times:
        song_name = dstore.get_song_file_from_id(song_id)

        #if query_duration > 5 and db_duration > 5:
        matches_to_return.append((song_name, query_start_time, db_start_time))
    return matches_to_return

def is_match(f1, f2):
    """Returns True if f1 matches to f2.

    Currently works by comparing the number of matched fingerprints
    that occur at the same offset difference between the two songs.

    If this number is greater than MATCH_THRESHOLD, then it returns
    True.

    Wow, code reuse. Remind yourself to kick dan.
    Not very :herb:

    TODO: WHY IS THIS SO SLOW

    """
    mono1, mono2 = read_audio.get_mono(f1), read_audio.get_mono(f2)
    prints1, prints2 = [dict(fingerprinting.get_fingerprints(m))
                        for m in (mono1, mono2)]
    shared_hashes = set(prints1.keys()).intersection(set(prints2.keys()))

    # {offset_diff -> [(md5, offset1, offset2), ...]}
    offset_dict = defaultdict(list)
    match_counter = defaultdict(int)

    max_count = 0
    offset_difference = 0

    for h in shared_hashes:
        offset1, offset2 = prints1[h], prints2[h]
        offset_diff = abs(offset1 - offset2)
        offset_dict[offset_diff].append((h, offset1, offset2))
        match_counter[offset_diff] += 1
        count = match_counter[offset_diff]
        if count > max_count:
            max_count = count
            offset_difference = offset_diff

    if max_count > MATCH_THRESHOLD:
        hashes = match_counter[offset_difference]
        start1 = min(hashes, key=lambda t: t[1])[1]
        start2 = min(hashes, key=lambda t: t[2])[2]

        return (start1, start2)

    else:
        return False

def print_match(audio_1_path, match_data):
    """Prints matches according to black-box spec
    """

    for match in match_data:
        audio_2_path = match[0]
        time_1 = match[1]
        time_2 = match[2]

        # python will automatically include spaces between comma-deliniated elements
        print "MATCH", basename(audio_1_path).lstrip(), basename(audio_2_path).lstrip(), time_1, time_2
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import subprocess
import os
import csv

class base_method:
    def __init__(self, method_name):
        self.java_name = ''
        self.method_name = method_name
        self.begin_line = 0
        self.end_line = 0
        self.buggy = False
        self.covered = False

    @property
    def java_name(self):
        return self.java_name

    @java_name.setter
    def java_name(self, java_name):
        self.java_name = java_name
        return

    @property
    def begin_line(self):
        return self.begin_line

    @begin_line.setter
    def begin_line(self, begin_line):
        self.begin_line = begin_line
        return

    @property
    def end_line(self):
        return self.end_line

    @end_line.setter
    def end_line(self, end_line):
        self.end_line = end_line
        return

    def is_buggy(self):
        self.buggy = False
        return

    def is_covered(self):
        result = subprocess.check_output('find ./coverage_report/coverage-report_converted/ -name ' + self.java_name.split('/')[-1].replace('.java', '.csv'), shell=True)
        result = result.replace('\r', '')
        result_list = result.split('\n')
        result_list.pop()
        coverage_report_name = ''
        for coverage_report_candidate in result_list:
            if self.java_name.replace('.java', '.csv') in coverage_report_candidate:
                coverage_report_name = coverage_report_candidate
                break
        if coverage_report_name == '':
            self.covered = False
        else:
            for line_num in range(self.begin_line, self.end_line+1):
                print line_num
                try:
                    result = subprocess.check_output('grep "$' + str(line_num) + '|" ' + coverage_report_name, shell=True)
                    self.covered = True
                    break
                except subprocess.CalledProcessError:
                    self.covered = False
                    continue
        return
<commit_msg>Added is_buggy() method in base_method class.<commit_after># -*- coding: utf-8 -*-
import sys
import subprocess
import os
import csv

class base_method:
    def __init__(self, method_name):
        self.java_name = ''
        self.method_name = method_name
        self.begin_line = 0
        self.end_line = 0
        self.buggy = False
        self.covered = False

    @property
    def java_name(self):
        return self.java_name

    @java_name.setter
    def java_name(self, java_name):
        self.java_name = java_name
        return

    @property
    def begin_line(self):
        return self.begin_line

    @begin_line.setter
    def begin_line(self, begin_line):
        self.begin_line = begin_line
        return

    @property
    def end_line(self):
        return self.end_line

    @end_line.setter
    def end_line(self, end_line):
        self.end_line = end_line
        return

    def is_buggy(self, git_tag_name):
        try:
            result = subprocess.check_output('grep "' + self.method_name + '" ./bug_location/bug_location_info.csv | grep "' + git_tag_name + '"', shell=True)
            result = result.replace('\r', '')
            result_list = result.split('\n')
            result_list.pop()
            for bug_info in result_list:
                bug_begin_line = int(bug_info.split(',')[-2])
                bug_end_line = int(bug_info.split(',')[-3])
                if ((bug_begin_line > self.end_line) or (bug_end_line < self.begin_line)):
                    self.buggy = False
                else:
                    self.buggy = True
                    break
        except subprocess.CalledProcessError:
            self.buggy = False
        return

    def is_covered(self):
        result = subprocess.check_output('find ./coverage_report/coverage-report_converted/ -name ' + self.java_name.split('/')[-1].replace('.java', '.csv'), shell=True)
        result = result.replace('\r', '')
        result_list = result.split('\n')
        result_list.pop()
        coverage_report_name = ''
        for coverage_report_candidate in result_list:
            if self.java_name.replace('.java', '.csv') in coverage_report_candidate:
                coverage_report_name = coverage_report_candidate
                break
        if coverage_report_name == '':
            self.covered = False
        else:
            for line_num in range(self.begin_line, self.end_line+1):
                print line_num
                try:
                    result = subprocess.check_output('grep "$' + str(line_num) + '|" ' + coverage_report_name, shell=True)
                    self.covered = True
                    break
                except subprocess.CalledProcessError:
                    self.covered = False
                    continue
        return
<|endoftext|>


<commit_before>try:
    import ure as re
except ImportError:
    try:
        import re
    except ImportError:
        print("SKIP")
        raise SystemExit

r = re.compile(".+")
m = r.match("abc")
print(m.group(0))
try:
    m.group(1)
except IndexError:
    print("IndexError")

# conversion of re and match to string
str(r)
str(m)

r = re.compile("(.+)1")
m = r.match("xyz781")
print(m.group(0))
print(m.group(1))
try:
    m.group(2)
except IndexError:
    print("IndexError")

r = re.compile("[a-cu-z]")
m = r.match("a")
print(m.group(0))
m = r.match("z")
print(m.group(0))
m = r.match("d")
print(m)
m = r.match("A")
print(m)
print("===")

r = re.compile("[^a-cu-z]")
m = r.match("a")
print(m)
m = r.match("z")
print(m)
m = r.match("d")
print(m.group(0))
m = r.match("A")
print(m.group(0))

r = re.compile("o+")
m = r.search("foobar")
print(m.group(0))
try:
    m.group(1)
except IndexError:
    print("IndexError")

m = re.match(".*", "foo")
print(m.group(0))

m = re.search("w.r", "hello world")
print(m.group(0))

m = re.match('a+?', 'ab');  print(m.group(0))
m = re.match('a*?', 'ab');  print(m.group(0))
m = re.match('^ab$', 'ab'); print(m.group(0))
m = re.match('a|b', 'b');   print(m.group(0))
m = re.match('a|b|c', 'c'); print(m.group(0))

# Case where anchors fail to match
r = re.compile("^b|b$")
m = r.search("abc")
print(m)

try:
    re.compile("*")
except:
    print("Caught invalid regex")

# bytes objects
m = re.match(rb'a+?', b'ab');  print(m.group(0))
<commit_msg>tests/extmod: Add test for '-' in character class in regex.<commit_after>try:
    import ure as re
except ImportError:
    try:
        import re
    except ImportError:
        print("SKIP")
        raise SystemExit

r = re.compile(".+")
m = r.match("abc")
print(m.group(0))
try:
    m.group(1)
except IndexError:
    print("IndexError")

# conversion of re and match to string
str(r)
str(m)

r = re.compile("(.+)1")
m = r.match("xyz781")
print(m.group(0))
print(m.group(1))
try:
    m.group(2)
except IndexError:
    print("IndexError")

r = re.compile("[a-cu-z]")
m = r.match("a")
print(m.group(0))
m = r.match("z")
print(m.group(0))
m = r.match("d")
print(m)
m = r.match("A")
print(m)
print("===")

r = re.compile("[^a-cu-z]")
m = r.match("a")
print(m)
m = r.match("z")
print(m)
m = r.match("d")
print(m.group(0))
m = r.match("A")
print(m.group(0))
print("===")

# '-' character within character class block
print(re.match("[-a]+", "-a]d").group(0))
print(re.match("[a-]+", "-a]d").group(0))
print("===")

r = re.compile("o+")
m = r.search("foobar")
print(m.group(0))
try:
    m.group(1)
except IndexError:
    print("IndexError")

m = re.match(".*", "foo")
print(m.group(0))

m = re.search("w.r", "hello world")
print(m.group(0))

m = re.match('a+?', 'ab');  print(m.group(0))
m = re.match('a*?', 'ab');  print(m.group(0))
m = re.match('^ab$', 'ab'); print(m.group(0))
m = re.match('a|b', 'b');   print(m.group(0))
m = re.match('a|b|c', 'c'); print(m.group(0))

# Case where anchors fail to match
r = re.compile("^b|b$")
m = r.search("abc")
print(m)

try:
    re.compile("*")
except:
    print("Caught invalid regex")

# bytes objects
m = re.match(rb'a+?', b'ab');  print(m.group(0))
<|endoftext|>


<commit_before># Copyright (C) 2015 Research and Innovation Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2017 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bitstring import Bits
from ryu.ofproto import ofproto_v1_3 as ofp
from ryu.ofproto import ofproto_v1_3_parser as parser
from ryu.lib import addrconv

class FakeOFTable(object):
    """Fake OFTable is a virtual openflow pipeline used for testing openflow controllers.

    The tables are populated using apply_ofmsgs and can be queried with
    is_ouput.
    """

    def __init__(self, num_tables):
        self.tables = [[] for _ in range(0, num_tables)]

    def _apply_flowmod(self, ofmsg):
        table_id = ofmsg.table_id
        if table_id == ofp.OFPTT_ALL or table_id is None:
            tables = self.tables
        else:
            tables = [self.tables[table_id]]
        flowmod = FlowMod(ofmsg)
        for table in tables:
            if ofmsg.command == ofp.OFPFC_ADD:
                # From the 1.3 spec, section 6.4:
                # For add requests (OFPFC_ADD) with the
                # OFPFF_CHECK_OVERLAP flag set, the switch must first
                # check for any overlapping flow entries in the
                # requested table.  Two flow entries overlap if a
                # single packet may match both, and both flow entries
                # have the same priority, but the two flow entries
                # don't have the exact same match.  If an overlap
                # conflict exists between an existing flow entry and
                # the add request, the switch must refuse the addition
                # and respond with an ofp_error_msg with
                # OFPET_FLOW_MOD_FAILED type and OFPFMFC_OVERLAP code.
                #
                # Without the check overlap flag it seems like it is
                # possible that we can have overlapping flow table
                # entries which will cause ambiguous behaviour. This is
                # obviously unnacceptable so we will assume this is
                # always set
                add = True
                for fte in table:
                    if flowmod.fte_matches(fte, strict=True):
                        table.remove(fte)
                        break
                    elif flowmod.overlaps(fte):
                        add = False
                        break
                if add:
                    table.append(flowmod)
            elif ofmsg.command == ofp.OFPFC_DELETE:
                removals = [fte for fte in table if flowmod.fte_matches(fte)]
                for fte in removals:
                    table.remove(fte)
            elif ofmsg.command == ofp.OFPFC_DELETE_STRICT:
                for fte in table:
                    if flowmod.fte_matches(fte, strict=True):
                        table.remove(fte)
                        break
            elif ofmsg.command == ofp.OFPFC_MODIFY:
                for fte in table:
                    if flowmod.fte_matches(fte):
                        fte.instructions = flowmod.instructions
            elif ofmsg.command == ofp.OFPFC_MODIFY_STRICT:
                for fte in table:
                    if flowmod.fte_matches(fte, strict=True):
                        fte.instructions = flowmod.instructions
                        break

    def apply_ofmsgs(self, ofmsgs):
        """This is used to update the fake flowtable.

        Adds, Deletes and modify flow modification messages are applied
        according to section 6.4 of the OpenFlow 1.3 specification."""
        for ofmsg in ofmsgs:
            if isinstance(ofmsg, parser.OFPFlowMod):
                self._apply_flowmod(ofmsg)
                self.sort_tables()

    def lookup(self, match):
        """Return the entries from flowmods that matches match.

        Searches each table in the pipeline for the entries that will be
        applied to the packet with fields represented by match.

        Arguments:
        match: a dictionary keyed by header field names with values.
                header fields not provided in match must be wildcarded for the
                entry to be considered matching.

        Returns: a list of the flowmods that will be applied to the packet
                represented by match
        """
        packet_dict = match.copy() # Packet headers may be modified
        instructions = []
        table_id = 0
        goto_table = True
        while goto_table:
            goto_table = False
            table = self.tables[table_id]
            matching_fte = None
            # find a matching flowmod
            for fte in table:
                if fte.pkt_matches(packet_dict):
                    matching_fte = fte
                    break
            # if a flowmod is found, make modifications to the match values and
            # determine if another lookup is necessary
            if matching_fte:
                for instruction in matching_fte.instructions:
                    instructions.append(instruction)
                    if instruction.type == ofp.OFPIT_GOTO_TABLE:
                        if table_id < instruction.table_id:
                            table_id = instruction.table_id
                            goto_table = True
                    elif instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                        for action in instruction.actions:
                            if action.type == ofp.OFPAT_SET_FIELD:
                                packet_dict[action.key] = action.value
        return instructions

    def is_output(self, match, port=None, vid=None):
        """Return true if packets with match fields is output to port with
        correct vlan.

        If port is none it will return true if output to any port (including
        special ports) regardless of vlan tag.

        If vid is none it will return true if output to specified port
        regardless of vlan tag.

        To specify the packet should be output without a vlan tag set the
        OFPVID_PRESENT bit in vid is 0.

        Arguments:
        Match: a dictionary keyed by header field names with values.
        """
        # vid_stack represents the packet's vlan stack, innermost label listed
        # first
        match_vid = match.get('vlan_vid', 0)
        if match_vid & ofp.OFPVID_PRESENT != 0:
            vid_stack = [match_vid]
        else:
            vid_stack = []

        instructions = self.lookup(match)

        for instruction in instructions:
            if instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                for action in instruction.actions:
                    if action.type == ofp.OFPAT_PUSH_VLAN:
                        vid_stack.append(ofp.OFPVID_PRESENT)
                    elif action.type == ofp.OFPAT_POP_VLAN:
                        vid_stack.pop()
                    elif action.type == ofp.OFPAT_SET_FIELD:
                        if action.key == 'vlan_vid':
                            vid_stack[-1] = action.value
                        else:
                            continue
                    elif action.type == ofp.OFPAT_OUTPUT:
                        if port is None:
                            return True
                        if action.port == port:
                            if vid is None:
                                return True
                            if vid & ofp.OFPVID_PRESENT == 0:
                                return not vid_stack
                            return vid_stack and vid == vid_stack[-1]

        return False

    def __str__(self):
        string = ''
        for table_id, table in enumerate(self.tables):
            string += '----- Table %u -----\n' % (table_id)
            string += '\n'.join([flowmod for flowmod in table])
        return string

    def sort_tables(self):
        """Sort flows in tables by priority order."""
        self.tables = [sorted(table, reverse=True) for table in self.tables]

class FlowMod(object):
    """Represents a flow modification message and its corresponding entry in
    the flow table.
    """
    MAC_MATCH_FIELDS = (
        'eth_src', 'eth_dst', 'arp_sha', 'arp_tha', 'ipv6_nd_sll',
        'ipv6_nd_tll'
        )
    IPV4_MATCH_FIELDS = ('ipv4_src', 'ipv4_dst', 'arp_spa', 'arp_tpa')
    IPV6_MATCH_FIELDS = ('ipv6_src', 'ipv6_dst', 'ipv6_nd_target')

    def __init__(self, flowmod):
        """flowmod is a ryu flow modification message object"""
        self.priority = flowmod.priority
        self.instructions = flowmod.instructions
        self.match_values = {}
        self.match_masks = {}
        self.out_port = None
        if ((flowmod.command == ofp.OFPFC_DELETE or flowmod.command == ofp.OFPFC_DELETE_STRICT) and
                flowmod.out_port != ofp.OFPP_ANY):
            self.out_port = flowmod.out_port

        for key, val in flowmod.match.items():
            if isinstance(val, tuple):
                val, mask = val
            else:
                mask = -1

            mask = self.match_to_bits(key, mask)
            val = self.match_to_bits(key, val) & mask
            self.match_values[key] = val
            self.match_masks[key] = mask

    def out_port_matches(self, other):
        """returns True if other has an output action to this flowmods
        output_port"""
        if self.out_port is None or self.out_port == ofp.OFPP_ANY:
            return True
        for instruction in other.instructions:
            if instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                for action in instruction.actions:
                    if action.type == ofp.OFPAT_OUTPUT:
                        if action.port == self.out_port:
                            return True
        return False

    def pkt_matches(self, pkt_dict):
        """returns True if pkt_dict matches this flow table entry.

        args:
            pkt_dict - a dictionary keyed by flow table match fields with
                values
        if an element is included in the flow table entry match fields but not
        in the pkt_dict that is assumed to indicate a failed match
        """

        # TODO: add cookie and out_group
        for key, val in self.match_values.items():
            if key not in pkt_dict:
                return False
            else:
                val_bits = self.match_to_bits(key, pkt_dict[key])
                if val_bits != (val & self.match_masks[key]):
                    return False
        return True

    def _matches_match(self, other):
        return (self.priority == other.priority and
                self.match_values == other.match_values and
                self.match_masks == other.match_masks)

    def fte_matches(self, other, strict=False):
        """returns True if the flow table entry other matches this flowmod.

        used for finding existing flow table entries that match with this
        flowmod.

        args:
            other - a flowmod object
            strict (bool) - whether to use strict matching (as defined in
                of1.3 specification section 6.4)
        """
        if not self.out_port_matches(other):
            return False
        if strict:
            return self._matches_match(other)
        for key, val in self.match_values.items():
            if key not in other.match_values:
                return False
            else:
                if other.match_values[key] & self.match_masks[key] != val:
                    return False
        return True

    def overlaps(self, other):
        """ returns True if any packet can match both self and other."""
        # This is different from the matches method as matches assumes an
        # undefined field is a failed match. In this case an undefined field is
        # potentially an overlap and therefore is considered success
        if other.priority != self.priority:
            return False
        for key, val in self.match_values.items():
            if key in other.match_values:
                if val & other.match_masks[key] != other.match_values[key]:
                    return False
                if other.match_values[key] & self.match_masks[key] != val:
                    return False
        return True

    def match_to_bits(self, key, val):
        """convert match fields and masks to bits objects.

        this allows for masked matching. Converting all match fields to the
        same object simplifies things (eg __str__).
        """
        if isinstance(val, Bits):
            return val

        def _val_to_bits(conv, val, length):
            if val is -1:
                return Bits(int=-1, length=length)
            return Bits(bytes=conv(val), length=length)

        if key in self.MAC_MATCH_FIELDS:
            return _val_to_bits(addrconv.mac.text_to_bin, val, 48)
        elif key in self.IPV4_MATCH_FIELDS:
            return _val_to_bits(addrconv.ipv4.text_to_bin, val, 32)
        elif key in self.IPV6_MATCH_FIELDS:
            return _val_to_bits(addrconv.ipv6.text_to_bin, val, 128)
        else:
            val = Bits(int=int(val), length=64)
        return val

    def __lt__(self, other):
        return self.priority < other.priority

    def __eq__(self, other):
        return (self._matches_match(other) and
                self.out_port == other.out_port and
                self.instructions == other.instructions)

    def __str__(self):
        string = 'priority: {0}'.format(self.priority)
        for key, val in self.match_values.items():
            mask = self.match_masks[key]
            string += ' {0}: {1}'.format(key, val)
            if mask.int != -1:
                string += '/{0}'.format(mask)
        string += ' Instructions: {0}'.format(str(self.instructions))
        return string
<commit_msg>pylint.<commit_after># Copyright (C) 2015 Research and Innovation Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2017 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bitstring import Bits
from ryu.ofproto import ofproto_v1_3 as ofp
from ryu.ofproto import ofproto_v1_3_parser as parser
from ryu.lib import addrconv

class FakeOFTable(object):
    """Fake OFTable is a virtual openflow pipeline used for testing openflow controllers.

    The tables are populated using apply_ofmsgs and can be queried with
    is_ouput.
    """

    def __init__(self, num_tables):
        self.tables = [[] for _ in range(0, num_tables)]

    def _apply_flowmod(self, ofmsg):

        def _add(table, flowmod):
            # From the 1.3 spec, section 6.4:
            # For add requests (OFPFC_ADD) with the
            # OFPFF_CHECK_OVERLAP flag set, the switch must first
            # check for any overlapping flow entries in the
            # requested table.  Two flow entries overlap if a
            # single packet may match both, and both flow entries
            # have the same priority, but the two flow entries
            # don't have the exact same match.  If an overlap
            # conflict exists between an existing flow entry and
            # the add request, the switch must refuse the addition
            # and respond with an ofp_error_msg with
            # OFPET_FLOW_MOD_FAILED type and OFPFMFC_OVERLAP code.
            #
            # Without the check overlap flag it seems like it is
            # possible that we can have overlapping flow table
            # entries which will cause ambiguous behaviour. This is
            # obviously unnacceptable so we will assume this is
            # always set
            add = True
            for fte in table:
                if flowmod.fte_matches(fte, strict=True):
                    table.remove(fte)
                    break
                elif flowmod.overlaps(fte):
                    add = False
                    break
            if add:
                table.append(flowmod)

        def _del(table, flowmod):
            removals = [fte for fte in table if flowmod.fte_matches(fte)]
            for fte in removals:
                table.remove(fte)

        def _del_strict(table, flowmod):
            for fte in table:
                if flowmod.fte_matches(fte, strict=True):
                    table.remove(fte)
                    break

        def _modify(table, flowmod):
            for fte in table:
                if flowmod.fte_matches(fte):
                    fte.instructions = flowmod.instructions

        def _modify_strict(table, flowmod):
            for fte in table:
                if flowmod.fte_matches(fte, strict=True):
                    fte.instructions = flowmod.instructions
                    break

        _flowmod_handlers = {
            ofp.OFPFC_ADD: _add,
            ofp.OFPFC_DELETE: _del,
            ofp.OFPFC_DELETE_STRICT: _del_strict,
            ofp.OFPFC_MODIFY: _modify,
            ofp.OFPFC_MODIFY_STRICT: _modify_strict,
        }

        table_id = ofmsg.table_id
        if table_id == ofp.OFPTT_ALL or table_id is None:
            tables = self.tables
        else:
            tables = [self.tables[table_id]]
        flowmod = FlowMod(ofmsg)

        for table in tables:
            _flowmod_handlers[ofmsg.command](table, flowmod)

    def apply_ofmsgs(self, ofmsgs):
        """This is used to update the fake flowtable.

        Adds, Deletes and modify flow modification messages are applied
        according to section 6.4 of the OpenFlow 1.3 specification."""
        for ofmsg in ofmsgs:
            if isinstance(ofmsg, parser.OFPFlowMod):
                self._apply_flowmod(ofmsg)
                self.sort_tables()

    def lookup(self, match):
        """Return the entries from flowmods that matches match.

        Searches each table in the pipeline for the entries that will be
        applied to the packet with fields represented by match.

        Arguments:
        match: a dictionary keyed by header field names with values.
                header fields not provided in match must be wildcarded for the
                entry to be considered matching.

        Returns: a list of the flowmods that will be applied to the packet
                represented by match
        """
        packet_dict = match.copy() # Packet headers may be modified
        instructions = []
        table_id = 0
        goto_table = True
        while goto_table:
            goto_table = False
            table = self.tables[table_id]
            matching_fte = None
            # find a matching flowmod
            for fte in table:
                if fte.pkt_matches(packet_dict):
                    matching_fte = fte
                    break
            # if a flowmod is found, make modifications to the match values and
            # determine if another lookup is necessary
            if matching_fte:
                for instruction in matching_fte.instructions:
                    instructions.append(instruction)
                    if instruction.type == ofp.OFPIT_GOTO_TABLE:
                        if table_id < instruction.table_id:
                            table_id = instruction.table_id
                            goto_table = True
                    elif instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                        for action in instruction.actions:
                            if action.type == ofp.OFPAT_SET_FIELD:
                                packet_dict[action.key] = action.value
        return instructions

    def is_output(self, match, port=None, vid=None):
        """Return true if packets with match fields is output to port with
        correct vlan.

        If port is none it will return true if output to any port (including
        special ports) regardless of vlan tag.

        If vid is none it will return true if output to specified port
        regardless of vlan tag.

        To specify the packet should be output without a vlan tag set the
        OFPVID_PRESENT bit in vid is 0.

        Arguments:
        Match: a dictionary keyed by header field names with values.
        """
        # vid_stack represents the packet's vlan stack, innermost label listed
        # first
        match_vid = match.get('vlan_vid', 0)
        if match_vid & ofp.OFPVID_PRESENT != 0:
            vid_stack = [match_vid]
        else:
            vid_stack = []

        instructions = self.lookup(match)

        for instruction in instructions:
            if instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                for action in instruction.actions:
                    if action.type == ofp.OFPAT_PUSH_VLAN:
                        vid_stack.append(ofp.OFPVID_PRESENT)
                    elif action.type == ofp.OFPAT_POP_VLAN:
                        vid_stack.pop()
                    elif action.type == ofp.OFPAT_SET_FIELD:
                        if action.key == 'vlan_vid':
                            vid_stack[-1] = action.value
                        else:
                            continue
                    elif action.type == ofp.OFPAT_OUTPUT:
                        if port is None:
                            return True
                        if action.port == port:
                            if vid is None:
                                return True
                            if vid & ofp.OFPVID_PRESENT == 0:
                                return not vid_stack
                            return vid_stack and vid == vid_stack[-1]

        return False

    def __str__(self):
        string = ''
        for table_id, table in enumerate(self.tables):
            string += '----- Table %u -----\n' % (table_id)
            string += '\n'.join([flowmod for flowmod in table])
        return string

    def sort_tables(self):
        """Sort flows in tables by priority order."""
        self.tables = [sorted(table, reverse=True) for table in self.tables]

class FlowMod(object):
    """Represents a flow modification message and its corresponding entry in
    the flow table.
    """
    MAC_MATCH_FIELDS = (
        'eth_src', 'eth_dst', 'arp_sha', 'arp_tha', 'ipv6_nd_sll',
        'ipv6_nd_tll'
        )
    IPV4_MATCH_FIELDS = ('ipv4_src', 'ipv4_dst', 'arp_spa', 'arp_tpa')
    IPV6_MATCH_FIELDS = ('ipv6_src', 'ipv6_dst', 'ipv6_nd_target')

    def __init__(self, flowmod):
        """flowmod is a ryu flow modification message object"""
        self.priority = flowmod.priority
        self.instructions = flowmod.instructions
        self.match_values = {}
        self.match_masks = {}
        self.out_port = None
        if ((flowmod.command == ofp.OFPFC_DELETE or flowmod.command == ofp.OFPFC_DELETE_STRICT) and
                flowmod.out_port != ofp.OFPP_ANY):
            self.out_port = flowmod.out_port

        for key, val in flowmod.match.items():
            if isinstance(val, tuple):
                val, mask = val
            else:
                mask = -1

            mask = self.match_to_bits(key, mask)
            val = self.match_to_bits(key, val) & mask
            self.match_values[key] = val
            self.match_masks[key] = mask

    def out_port_matches(self, other):
        """returns True if other has an output action to this flowmods
        output_port"""
        if self.out_port is None or self.out_port == ofp.OFPP_ANY:
            return True
        for instruction in other.instructions:
            if instruction.type == ofp.OFPIT_APPLY_ACTIONS:
                for action in instruction.actions:
                    if action.type == ofp.OFPAT_OUTPUT:
                        if action.port == self.out_port:
                            return True
        return False

    def pkt_matches(self, pkt_dict):
        """returns True if pkt_dict matches this flow table entry.

        args:
            pkt_dict - a dictionary keyed by flow table match fields with
                values
        if an element is included in the flow table entry match fields but not
        in the pkt_dict that is assumed to indicate a failed match
        """

        # TODO: add cookie and out_group
        for key, val in self.match_values.items():
            if key not in pkt_dict:
                return False
            else:
                val_bits = self.match_to_bits(key, pkt_dict[key])
                if val_bits != (val & self.match_masks[key]):
                    return False
        return True

    def _matches_match(self, other):
        return (self.priority == other.priority and
                self.match_values == other.match_values and
                self.match_masks == other.match_masks)

    def fte_matches(self, other, strict=False):
        """returns True if the flow table entry other matches this flowmod.

        used for finding existing flow table entries that match with this
        flowmod.

        args:
            other - a flowmod object
            strict (bool) - whether to use strict matching (as defined in
                of1.3 specification section 6.4)
        """
        if not self.out_port_matches(other):
            return False
        if strict:
            return self._matches_match(other)
        for key, val in self.match_values.items():
            if key not in other.match_values:
                return False
            else:
                if other.match_values[key] & self.match_masks[key] != val:
                    return False
        return True

    def overlaps(self, other):
        """ returns True if any packet can match both self and other."""
        # This is different from the matches method as matches assumes an
        # undefined field is a failed match. In this case an undefined field is
        # potentially an overlap and therefore is considered success
        if other.priority != self.priority:
            return False
        for key, val in self.match_values.items():
            if key in other.match_values:
                if val & other.match_masks[key] != other.match_values[key]:
                    return False
                if other.match_values[key] & self.match_masks[key] != val:
                    return False
        return True

    def match_to_bits(self, key, val):
        """convert match fields and masks to bits objects.

        this allows for masked matching. Converting all match fields to the
        same object simplifies things (eg __str__).
        """
        if isinstance(val, Bits):
            return val

        def _val_to_bits(conv, val, length):
            if val is -1:
                return Bits(int=-1, length=length)
            return Bits(bytes=conv(val), length=length)

        if key in self.MAC_MATCH_FIELDS:
            return _val_to_bits(addrconv.mac.text_to_bin, val, 48)
        elif key in self.IPV4_MATCH_FIELDS:
            return _val_to_bits(addrconv.ipv4.text_to_bin, val, 32)
        elif key in self.IPV6_MATCH_FIELDS:
            return _val_to_bits(addrconv.ipv6.text_to_bin, val, 128)
        else:
            val = Bits(int=int(val), length=64)
        return val

    def __lt__(self, other):
        return self.priority < other.priority

    def __eq__(self, other):
        return (self._matches_match(other) and
                self.out_port == other.out_port and
                self.instructions == other.instructions)

    def __str__(self):
        string = 'priority: {0}'.format(self.priority)
        for key, val in self.match_values.items():
            mask = self.match_masks[key]
            string += ' {0}: {1}'.format(key, val)
            if mask.int != -1:
                string += '/{0}'.format(mask)
        string += ' Instructions: {0}'.format(str(self.instructions))
        return string
<|endoftext|>


<commit_before>import itertools
import errno
import stat
import os
import multiprocessing

import llfuse

class MartyFSHandler(llfuse.Operations):

    """ Fuse FS for a Marty Tree.
    """

    DEFAULT_MODE = 0o555
    DEFAULT_UID = 0
    DEFAULT_GID = 0

    def __init__(self, storage, root_tree):
        super(MartyFSHandler, self).__init__()
        self.storage = storage
        self.inodes = {}
        self.fd = {}
        self.inodes_index = itertools.count(llfuse.ROOT_INODE)
        self.fd_index = itertools.count()

        # First item registered will be the root tree
        self._register_item({'type': 'tree',
                             'tree': root_tree,
                             'mode': MartyFSHandler.DEFAULT_MODE,
                             'uid': MartyFSHandler.DEFAULT_UID,
                             'gid': MartyFSHandler.DEFAULT_GID})

    def _register_item(self, item):
        inode = next(self.inodes_index)

        if item.get('type') == 'tree' and 'ref' in item:
            item['tree'] = self.storage.get_tree(item['ref'])
        elif item.get('type') == 'blob' and 'ref' in item:
            item['blob'] = self.storage.get_blob(item['ref'])

        self.inodes[inode] = item

        return inode

    def getattr(self, inode, ctx=None):
        attrs = self.inodes.get(inode)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)  # FIXME

        if attrs.get('type') == 'tree':
            mode_filetype = stat.S_IFDIR
        elif attrs.get('type') == 'blob':
            mode_filetype = stat.S_IFREG
        elif attrs.get('filetype') == 'link':
            mode_filetype = stat.S_IFLNK
        elif attrs.get('filetype') == 'fifo':
            mode_filetype = stat.S_IFIFO
        else:
            raise llfuse.FUSEError(errno.ENOENT)  # FIXME

        entry = llfuse.EntryAttributes()
        entry.st_mode = mode_filetype | attrs.get('mode', MartyFSHandler.DEFAULT_MODE)

        if attrs.get('type') == 'blob' and 'ref' in attrs:
            entry.st_size = self.storage.size(attrs['ref'])
        else:
            entry.st_size = 0

        stamp = int(1438467123.985654 * 1e9)
        entry.st_atime_ns = stamp
        entry.st_ctime_ns = stamp
        entry.st_mtime_ns = stamp
        entry.st_gid = 0
        entry.st_uid = 0
        entry.st_ino = inode

        return entry

    def lookup(self, parent_inode, name, ctx=None):
        attrs = self.inodes.get(parent_inode)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)
        elif attrs['type'] != 'tree':
            raise llfuse.FUSEError(errno.ENOENT)
        elif name not in attrs['tree']:
            raise llfuse.FUSEError(errno.ENOENT)

        if 'inode' not in attrs['tree'][name]:
            inode = self._register_item(attrs['tree'][name])
            attrs['tree'][name]['inode'] = inode

        return self.getattr(attrs['tree'][name]['inode'])

    def opendir(self, inode, ctx):
        return inode

    def readdir(self, fh, offset):
        # fh is actually the inode number for directories
        attrs = self.inodes.get(fh)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)
        elif attrs['type'] != 'tree':
            raise llfuse.FUSEError(errno.ENOENT)

        for i, (name, item) in enumerate(attrs['tree'].items()):
            if offset > i:
                continue

            if 'inode' not in item:
                inode = self._register_item(item)
                item['inode'] = inode

            yield (name, self.getattr(item['inode']), i + 1)

    def readlink(self, inode, ctx):
        attrs = self.inodes.get(inode)
        if 'link' not in attrs:
            raise llfuse.FUSEError(errno.ENOENT)

        return attrs['link'].encode('utf8')

    def open(self, inode, flags, ctx):
        attrs = self.inodes.get(inode)
        if 'blob' not in attrs:
            raise llfuse.FUSEError(errno.ENOENT)

        fh = next(self.fd_index)
        self.fd[fh] = attrs['blob'].to_file()

        return fh

    def read(self, fh, offset, size):
        fobj = self.fd[fh]
        fobj.seek(offset, os.SEEK_SET)
        return fobj.read(size)

    def release(self, fh):
        print(fh)
        if fh in self.fd:
            self.fd[fh].close()
            del self.fd[fh]

class MartyFS(object):

    def __init__(self, storage, root_tree, mountpoint):
        self.storage = storage
        self.root_tree = root_tree
        self.mountpoint = mountpoint
        self.llfuse_worker = None

    def _llfuse_worker(self, fs, mountpoint, init_event):
        fuse_options = set(llfuse.default_options)
        fuse_options.add('fsname=marty')
        llfuse.init(fs, mountpoint, fuse_options)
        init_event.set()
        try:
            llfuse.main(workers=1)
        finally:
            llfuse.close()

    def mount(self):
        if self.llfuse_worker is None:
            marty_fs = MartyFSHandler(self.storage, self.root_tree)
            init_event = multiprocessing.Event()
            self.llfuse_worker = multiprocessing.Process(target=self._llfuse_worker,
                                                         args=(marty_fs, self.mountpoint, init_event))
            self.llfuse_worker.start()
            init_event.wait()
        else:
            raise RuntimeError('Filesystem already mounted')

    def umount(self):
        if self.llfuse_worker is not None:
            self.llfuse_worker.terminate()
            self.llfuse_worker.join()
        else:
            raise RuntimeError('Filesystem is not mounted')
<commit_msg>operations: fix encoding error<commit_after>import itertools
import errno
import stat
import os
import multiprocessing

import llfuse

class MartyFSHandler(llfuse.Operations):

    """ Fuse FS for a Marty Tree.
    """

    DEFAULT_MODE = 0o555
    DEFAULT_UID = 0
    DEFAULT_GID = 0

    def __init__(self, storage, root_tree):
        super(MartyFSHandler, self).__init__()
        self.storage = storage
        self.inodes = {}
        self.fd = {}
        self.inodes_index = itertools.count(llfuse.ROOT_INODE)
        self.fd_index = itertools.count()

        # First item registered will be the root tree
        self._register_item({'type': 'tree',
                             'tree': root_tree,
                             'mode': MartyFSHandler.DEFAULT_MODE,
                             'uid': MartyFSHandler.DEFAULT_UID,
                             'gid': MartyFSHandler.DEFAULT_GID})

    def _register_item(self, item):
        inode = next(self.inodes_index)

        if item.get('type') == 'tree' and 'ref' in item:
            item['tree'] = self.storage.get_tree(item['ref'])
        elif item.get('type') == 'blob' and 'ref' in item:
            item['blob'] = self.storage.get_blob(item['ref'])

        self.inodes[inode] = item

        return inode

    def getattr(self, inode, ctx=None):
        attrs = self.inodes.get(inode)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)  # FIXME

        if attrs.get('type') == 'tree':
            mode_filetype = stat.S_IFDIR
        elif attrs.get('type') == 'blob':
            mode_filetype = stat.S_IFREG
        elif attrs.get('filetype') == 'link':
            mode_filetype = stat.S_IFLNK
        elif attrs.get('filetype') == 'fifo':
            mode_filetype = stat.S_IFIFO
        else:
            raise llfuse.FUSEError(errno.ENOENT)  # FIXME

        entry = llfuse.EntryAttributes()
        entry.st_mode = mode_filetype | attrs.get('mode', MartyFSHandler.DEFAULT_MODE)

        if attrs.get('type') == 'blob' and 'ref' in attrs:
            entry.st_size = self.storage.size(attrs['ref'])
        else:
            entry.st_size = 0

        stamp = int(1438467123.985654 * 1e9)
        entry.st_atime_ns = stamp
        entry.st_ctime_ns = stamp
        entry.st_mtime_ns = stamp
        entry.st_gid = 0
        entry.st_uid = 0
        entry.st_ino = inode

        return entry

    def lookup(self, parent_inode, name, ctx=None):
        attrs = self.inodes.get(parent_inode)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)
        elif attrs['type'] != 'tree':
            raise llfuse.FUSEError(errno.ENOENT)
        elif name not in attrs['tree']:
            raise llfuse.FUSEError(errno.ENOENT)

        if 'inode' not in attrs['tree'][name]:
            inode = self._register_item(attrs['tree'][name])
            attrs['tree'][name]['inode'] = inode

        return self.getattr(attrs['tree'][name]['inode'])

    def opendir(self, inode, ctx):
        return inode

    def readdir(self, fh, offset):
        # fh is actually the inode number for directories
        attrs = self.inodes.get(fh)
        if attrs is None:
            raise llfuse.FUSEError(errno.ENOENT)
        elif attrs['type'] != 'tree':
            raise llfuse.FUSEError(errno.ENOENT)

        for i, (name, item) in enumerate(attrs['tree'].items()):
            if offset > i:
                continue

            if 'inode' not in item:
                inode = self._register_item(item)
                item['inode'] = inode

            yield (name, self.getattr(item['inode']), i + 1)

    def readlink(self, inode, ctx):
        attrs = self.inodes.get(inode)
        if 'link' not in attrs:
            raise llfuse.FUSEError(errno.ENOENT)

        return attrs['link']

    def open(self, inode, flags, ctx):
        attrs = self.inodes.get(inode)
        if 'blob' not in attrs:
            raise llfuse.FUSEError(errno.ENOENT)

        fh = next(self.fd_index)
        self.fd[fh] = attrs['blob'].to_file()

        return fh

    def read(self, fh, offset, size):
        fobj = self.fd[fh]
        fobj.seek(offset, os.SEEK_SET)
        return fobj.read(size)

    def release(self, fh):
        print(fh)
        if fh in self.fd:
            self.fd[fh].close()
            del self.fd[fh]

class MartyFS(object):

    def __init__(self, storage, root_tree, mountpoint):
        self.storage = storage
        self.root_tree = root_tree
        self.mountpoint = mountpoint
        self.llfuse_worker = None

    def _llfuse_worker(self, fs, mountpoint, init_event):
        fuse_options = set(llfuse.default_options)
        fuse_options.add('fsname=marty')
        llfuse.init(fs, mountpoint, fuse_options)
        init_event.set()
        try:
            llfuse.main(workers=1)
        finally:
            llfuse.close()

    def mount(self):
        if self.llfuse_worker is None:
            marty_fs = MartyFSHandler(self.storage, self.root_tree)
            init_event = multiprocessing.Event()
            self.llfuse_worker = multiprocessing.Process(target=self._llfuse_worker,
                                                         args=(marty_fs, self.mountpoint, init_event))
            self.llfuse_worker.start()
            init_event.wait()
        else:
            raise RuntimeError('Filesystem already mounted')

    def umount(self):
        if self.llfuse_worker is not None:
            self.llfuse_worker.terminate()
            self.llfuse_worker.join()
        else:
            raise RuntimeError('Filesystem is not mounted')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import importlib
from flask_migrate import Migrate, MigrateCommand
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask_script import Manager

DEFAULT_EXTENSIONS = (
    "atomicpress.ext.importer",
    "atomicpress.ext.exporter",
    "atomicpress.ext.ftp",
    "atomicpress.ext.s3",
    "atomicpress.ext.prefill",
)

DEFAULT_MARKDOWN_EXTENSIONS = [
    "markdown.extensions.tables",
    "markdown.extensions.nl2br",
    "markdown.extensions.fenced_code",
    "markdown.extensions.headerid"
]

app = Flask(__name__)
app.config.update(dict(
    SQLALCHEMY_DATABASE_URI="",
    DEBUG=True,
    SECRET_KEY='',
    STATIC_URL='/static/',
    UPLOADS_URL='/uploads/',
    GIST_BACKEND_RENDERING=True,
    THEME="atomicpress.themes.minimal",
    EXTENSIONS=DEFAULT_EXTENSIONS,
    MARKDOWN_EXTENSIONS=DEFAULT_MARKDOWN_EXTENSIONS
))

settings_module = os.environ.get("ATOMICPRESS_SETTINGS")

if not settings_module:
    app.logger.warning("ATOMICPRESS_SETTINGS is empty, please specify a "
                       "settings file")

app.config.from_object(settings_module)

if "UPLOADS_PATH" not in app.config or \
        not os.path.isdir(app.config["UPLOADS_PATH"]):

    app.logger.error("UPLOADS_PATH is empty or is not a folder")
    exit()

db = SQLAlchemy()
db.init_app(app)

migrate = Migrate(app, db)

manager = Manager(app)

def setup(init_run=False):
    print("setup!")
    from atomicpress import models  # NOQA
    from atomicpress import views  # NOQA
    from atomicpress import filters  # NOQA
    from atomicpress import commands  # NOQA
    from atomicpress import context_processors  # NOQA

    manager.add_command('db', MigrateCommand)

    activate_extensions()

    # Activate theme
    theme = importlib.import_module(app.config["THEME"])
    activate_theme(theme.theme)

    if init_run:
        run()

def activate_extensions():
    for extension_module in app.config["EXTENSIONS"]:
        extension = importlib.import_module(extension_module)
        extension.setup()

def activate_theme(theme):
    app.register_blueprint(theme, url_prefix="")

def run():
    manager.run()
<commit_msg>Removed log<commit_after># -*- coding: utf-8 -*-

import os
import importlib
from flask_migrate import Migrate, MigrateCommand
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask_script import Manager

DEFAULT_EXTENSIONS = (
    "atomicpress.ext.importer",
    "atomicpress.ext.exporter",
    "atomicpress.ext.ftp",
    "atomicpress.ext.s3",
    "atomicpress.ext.prefill",
)

DEFAULT_MARKDOWN_EXTENSIONS = [
    "markdown.extensions.tables",
    "markdown.extensions.nl2br",
    "markdown.extensions.fenced_code",
    "markdown.extensions.headerid"
]

app = Flask(__name__)
app.config.update(dict(
    SQLALCHEMY_DATABASE_URI="",
    DEBUG=True,
    SECRET_KEY='',
    STATIC_URL='/static/',
    UPLOADS_URL='/uploads/',
    GIST_BACKEND_RENDERING=True,
    THEME="atomicpress.themes.minimal",
    EXTENSIONS=DEFAULT_EXTENSIONS,
    MARKDOWN_EXTENSIONS=DEFAULT_MARKDOWN_EXTENSIONS
))

settings_module = os.environ.get("ATOMICPRESS_SETTINGS")

if not settings_module:
    app.logger.warning("ATOMICPRESS_SETTINGS is empty, please specify a "
                       "settings file")

app.config.from_object(settings_module)

if "UPLOADS_PATH" not in app.config or \
        not os.path.isdir(app.config["UPLOADS_PATH"]):

    app.logger.error("UPLOADS_PATH is empty or is not a folder")
    exit()

db = SQLAlchemy()
db.init_app(app)

migrate = Migrate(app, db)

manager = Manager(app)

def setup(init_run=False):
    from atomicpress import models  # NOQA
    from atomicpress import views  # NOQA
    from atomicpress import filters  # NOQA
    from atomicpress import commands  # NOQA
    from atomicpress import context_processors  # NOQA

    manager.add_command('db', MigrateCommand)

    activate_extensions()

    # Activate theme
    theme = importlib.import_module(app.config["THEME"])
    activate_theme(theme.theme)

    if init_run:
        run()

def activate_extensions():
    for extension_module in app.config["EXTENSIONS"]:
        extension = importlib.import_module(extension_module)
        extension.setup()

def activate_theme(theme):
    app.register_blueprint(theme, url_prefix="")

def run():
    manager.run()
<|endoftext|>


<commit_before>"""
django-masquerade
"""
__version_info__ = {
    'major': 0,
    'minor': 1,
    'micro': 6,
    'releaselevel': 'final',
    'serial': 1
}

def get_version():
    """
    Return the formatted version information
    """
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()
<commit_msg>incremented version number for release<commit_after>"""
django-masquerade
"""
__version_info__ = {
    'major': 0,
    'minor': 1,
    'micro': 7,
    'releaselevel': 'final',
    'serial': 1
}

def get_version():
    """
    Return the formatted version information
    """
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# FILE: gists.py
# AUTHOR: pocari <caffelattenonsugar at gmail.com>
# License: MIT license

import urllib.request
import urllib.parse
import json

from denite import util
from .base import Base

GIST_BASE_URL = 'https://api.github.com/users/{}/gists'

class Source(Base):

    def __init__(self, vim):
        super().__init__(vim)

        self.name = 'gists'
        self.kind = 'open_browser'
        self.vars = {
            # TODO
        }
        self.api_params = {}

    def on_init(self, context):
        self.api_params = {
            # TODO
        }

    def gather_candidates(self, context):
        gists_url = self._build_url(context, self.api_params)
        gists = self._get_gists(gists_url)
        if gists:
            return [self._convert(r) for r in gists]

    def _build_url(self, context, params):
        args = dict(enumerate(context['args']))
        user_name = self._detect_github_user(args)
        url = GIST_BASE_URL.format(user_name)

        encoded_params = urllib.parse.urlencode(params)
        if encoded_params:
            url = '{}?{}'.format(url, encoded_params)

        util.debug(self.vim, url)
        return url

    def _detect_github_user(self, args):
        arg = args.get(0)
        if arg:
            user_name = arg
        else:
            u = self.vim.call('denite_gists#util#github_user')
            if u:
                user_name = u
            else:
                user_name = util.input(self.vim, context, 'Gist User: ')
        return user_name

    def _get_gists(self, url):
        try:
            with urllib.request.urlopen(url) as res:
                return json.loads(res.read().decode('utf-8'))
        except urllib.error.HTTPError as e:
            util.error(self.vim, 'Error {} api url:{}'.format(e.reason, url))

    def _convert(self, gist_info):
        return {
            'word': '{} ({})'.format(
                gist_info['description'],
                gist_info['updated_at']
            ),
            'url': gist_info['html_url']
        }
<commit_msg>delete debug line<commit_after># -*- coding: utf-8 -*-
# FILE: gists.py
# AUTHOR: pocari <caffelattenonsugar at gmail.com>
# License: MIT license

import urllib.request
import urllib.parse
import json

from denite import util
from .base import Base

GIST_BASE_URL = 'https://api.github.com/users/{}/gists'

class Source(Base):

    def __init__(self, vim):
        super().__init__(vim)

        self.name = 'gists'
        self.kind = 'open_browser'
        self.vars = {
            # TODO
        }
        self.api_params = {}

    def on_init(self, context):
        self.api_params = {
            # TODO
        }

    def gather_candidates(self, context):
        gists_url = self._build_url(context, self.api_params)
        gists = self._get_gists(gists_url)
        if gists:
            return [self._convert(r) for r in gists]

    def _build_url(self, context, params):
        args = dict(enumerate(context['args']))
        user_name = self._detect_github_user(args)
        url = GIST_BASE_URL.format(user_name)

        encoded_params = urllib.parse.urlencode(params)
        if encoded_params:
            url = '{}?{}'.format(url, encoded_params)

        return url

    def _detect_github_user(self, args):
        arg = args.get(0)
        if arg:
            user_name = arg
        else:
            u = self.vim.call('denite_gists#util#github_user')
            if u:
                user_name = u
            else:
                user_name = util.input(self.vim, context, 'Gist User: ')
        return user_name

    def _get_gists(self, url):
        try:
            with urllib.request.urlopen(url) as res:
                return json.loads(res.read().decode('utf-8'))
        except urllib.error.HTTPError as e:
            util.error(self.vim, 'Error {} api url:{}'.format(e.reason, url))

    def _convert(self, gist_info):
        return {
            'word': '{} ({})'.format(
                gist_info['description'],
                gist_info['updated_at']
            ),
            'url': gist_info['html_url']
        }
<|endoftext|>


<commit_before>#! /usr/bin/python
import os,sys,subprocess

def main(argv):

	print """
    ______           .____________                     _____
\______   \ ____   __| _/   _____/ ____ _____ ________/ ____\
 |       _// __ \ / __ |\_____  \ /    \\__  \\_  __ \   __\
 |    |   \  ___// /_/ |/        \   |  \/ __ \|  | \/|  |
 |____|_  /\___  >____ /_______  /___|  (____  /__|   |__|
        \/     \/     \/       \/     \/     \/
                                  redsnarf.ff0000@gmail.com
                                                  @redsnarf
"""

	print "[+]RedSnarf Quick and Dirty Installer"
	print "[+]Starting Installation Process"
	os.system("apt-get update")
	os.system("apt-get install python-ipy python2.7-dev libpq-dev python-dev libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libffi-dev --fix-missing")
	os.system("pip install netaddr")
	os.system("pip install termcolor")
	os.system("pip install python-ldap")

	if not os.path.isfile('/opt/creddump7/pwdump.py'):
		os.system("git clone https://github.com/Neohapsis/creddump7 /opt/creddump7")

	if not os.path.isfile('/usr/local/bin/secretsdump.py'):
		if not os.path.isfile('/tmp/impacket/setup.py'):
			os.system("git clone https://github.com/CoreSecurity/impacket.git /tmp/impacket")

		os.system("chmod 777 /tmp/impacket/setup.py")

		print "[+]All that's left to do is run python /tmp/impacket/setup.py install"

if __name__ == "__main__":
   main(sys.argv[1:])<commit_msg>Quick Setup File<commit_after>#! /usr/bin/python
import os,sys,subprocess

def main(argv):

	print """
    ______           .____________                     _____
\______   \ ____   __| _/   _____/ ____ _____ ________/ ____\
 |       _// __ \ / __ |\_____  \ /    \\__  \\_  __ \   __\
 |    |   \  ___// /_/ |/        \   |  \/ __ \|  | \/|  |
 |____|_  /\___  >____ /_______  /___|  (____  /__|   |__|
        \/     \/     \/       \/     \/     \/
                                  redsnarf.ff0000@gmail.com
                                                  @redsnarf
"""

	print "[+]RedSnarf Quick and Dirty Installer"
	print "[+]Starting Installation Process"
	os.system("apt-get update")
	os.system("apt-get install python-ipy python2.7-dev libpq-dev python-dev libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libffi-dev --fix-missing")
	os.system("pip install netaddr")
	os.system("pip install termcolor")
	os.system("pip install python-ldap")

	if not os.path.isfile('/opt/creddump7/pwdump.py'):
		os.system("git clone https://github.com/Neohapsis/creddump7 /opt/creddump7")

	if not os.path.isfile('/usr/local/bin/secretsdump.py'):
		if not os.path.isfile('/tmp/impacket/setup.py'):
			os.system("git clone https://github.com/CoreSecurity/impacket.git /tmp/impacket")

		os.system("chmod 777 /tmp/impacket/setup.py")

		print "[+]All that's left to do is run navigate to /tmp/impacket/ and run python setup.py install"
		print "[+]Bye"

if __name__ == "__main__":
   main(sys.argv[1:])<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################
#    Module Writen For Odoo, Open Source Management Solution
#
#    Copyright (c) 2011 Vauxoo - http://www.vauxoo.com
#    All Rights Reserved.
#    info Vauxoo (info@vauxoo.com)
#    coded by: moylop260@vauxoo.com
############################################################################

'''
    This file is used to add the field lang in runbot.build and the function
      that install and assign the language to the users in the instance
      generated.
'''

import logging

from openerp import api, fields, models, tools
from openerp.addons.runbot.runbot import run

_logger = logging.getLogger(__name__)

class RunbotBuild(models.Model):

    '''
    Inherit class runbot_build to add field to select the language &
      the function with a job
      to install and assign the language to users if this is captured
      too is added with an super the
      function create to assign the language from repo in the builds.
    '''

    _inherit = "runbot.build"

    lang = fields.Selection(
        tools.scan_languages(), 'Language',
        help='Language to change '
        'instance after of run test.', copy=True)

    @api.multi
    def cmd(self):
        """Return a list describing the command to start the build"""
        cmd, modules = super(RunbotBuild, self).cmd()
        for build in self:
            if build.lang and build.job == 'job_30_run':
                cmd.append("--load-language=%s" % (build.lang))
        return cmd, modules

    @api.one
    def update_lang(self):
        """Set lang to all users into '-all' database"""
        if self.lang:
            db_name = "%s-all" % self.dest
            # All odoo versions has openerp/release.py file
            sys.path.insert(0, build.server("openerp"))
            try:
                release =  __import__("release")
            finally:
                sys.path.pop(0)
            version = float("{main_version}.{secondary_version}".format(
                main_version=release.version_info[0],
                secondary_version=release.version_info[1]))

            if version < 7:
                run(['psql', db_name, '-c', "UPDATE res_users SET lang='%s';" %
                     (self.lang)])
            else:
                run(['psql', db_name, '-c', "UPDATE res_partner SET lang='%s' "
                     "WHERE id IN (SELECT partner_id FROM res_users);" %
                     (self.lang)])
        return True

    def job_30_run(self, cr, uid, build, lock_path, log_path):
        res = super(RunbotBuild, self).job_30_run(cr, uid, build,
                                                  lock_path, log_path)
        self.update_lang(cr, uid, build.id)
        return res

    @api.model
    def create(self, values):
        """
        This method set language from repo in the build.
        """
        if values.get('branch_id', False) and 'lang' not in values:
            lang = self.env['runbot.branch'].browse(
                values['branch_id']).repo_id.lang
            values.update({
                'lang': lang,
            })
        return super(RunbotBuild, self).create(values)
<commit_msg>[REF] runbot_language: Change validation of version<commit_after># -*- coding: utf-8 -*-
##############################################################
#    Module Writen For Odoo, Open Source Management Solution
#
#    Copyright (c) 2011 Vauxoo - http://www.vauxoo.com
#    All Rights Reserved.
#    info Vauxoo (info@vauxoo.com)
#    coded by: moylop260@vauxoo.com
############################################################################

'''
    This file is used to add the field lang in runbot.build and the function
      that install and assign the language to the users in the instance
      generated.
'''

import logging
import sys

from openerp import api, fields, models, tools
from openerp.addons.runbot.runbot import run

_logger = logging.getLogger(__name__)

class RunbotBuild(models.Model):

    '''
    Inherit class runbot_build to add field to select the language &
      the function with a job
      to install and assign the language to users if this is captured
      too is added with an super the
      function create to assign the language from repo in the builds.
    '''

    _inherit = "runbot.build"

    lang = fields.Selection(
        tools.scan_languages(), 'Language',
        help='Language to change '
        'instance after of run test.', copy=True)

    @api.multi
    def cmd(self):
        """Return a list describing the command to start the build"""
        cmd, modules = super(RunbotBuild, self).cmd()
        for build in self:
            if build.lang and build.job == 'job_30_run':
                cmd.append("--load-language=%s" % (build.lang))
        return cmd, modules

    @api.one
    def update_lang(self):
        """Set lang to all users into '-all' database"""
        if self.lang:
            db_name = "%s-all" % self.dest
            # All odoo versions has openerp/release.py file
            sys.path.insert(0, self.server("openerp"))
            try:
                release = __import__("release")
            finally:
                sys.path.pop(0)

            if release.version_info < (7, 0):
                # Old version used `res_users` table to set `lang`
                run(['psql', db_name, '-c',
                     "UPDATE res_users SET lang='{lang}';".format(
                         lang=self.lang)])
            else:
                # New version use `res_partner` table to set `lang`
                run(['psql', db_name, '-c',
                     "UPDATE res_partner SET lang='{lang}' ".format(
                         lang=self.lang) +
                     "WHERE id IN (SELECT partner_id FROM res_users);"])
        return True

    def job_30_run(self, cr, uid, build, lock_path, log_path):
        res = super(RunbotBuild, self).job_30_run(cr, uid, build,
                                                  lock_path, log_path)
        self.update_lang(cr, uid, build.id)
        return res

    @api.model
    def create(self, values):
        """
        This method set language from repo in the build.
        """
        if values.get('branch_id', False) and 'lang' not in values:
            lang = self.env['runbot.branch'].browse(
                values['branch_id']).repo_id.lang
            values.update({
                'lang': lang,
            })
        return super(RunbotBuild, self).create(values)
<|endoftext|>


<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Reference implementation of AugMix's data augmentation method in numpy."""
import augmentations
import numpy as np
from PIL import Image

# CIFAR-10 constants
MEAN = [0.4914, 0.4822, 0.4465]
STD = [0.2023, 0.1994, 0.2010]

def normalize(image):
  """Normalize input image channel-wise to zero mean and unit variance."""
  image = image.transpose(2, 0, 1)  # Switch to channel-first
  mean, std = np.array(MEAN), np.array(STD)
  image = (image - mean[:, None, None]) / std[:, None, None]
  return image.transpose(1, 2, 0)

def apply_op(image, op, severity):
  image = np.clip(image * 255., 0, 255).astype(np.uint8)
  pil_img = Image.fromarray(image)  # Convert to PIL.Image
  pil_img = op(pil_img, severity)
  return np.asarray(pil_img) / 255.

def augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.):
  """Perform AugMix augmentations and compute mixture.

  Args:
    image: Raw input image as float32 np.ndarray of shape (h, w, c)
    severity: Severity of underlying augmentation operators (between 1 to 10).
    width: Width of augmentation chain
    depth: Depth of augmentation chain. -1 enables stochastic depth uniformly
      from [1, 3]
    alpha: Probability coefficient for Beta and Dirichlet distributions.

  Returns:
    mixed: Augmented and mixed image.
  """
  ws = np.float32(
      np.random.dirichlet([alpha] * width))
  m = np.float32(np.random.beta(alpha, alpha))

  mix = np.zeros_like(image)
  for i in range(width):
    image_aug = image.copy()
    depth = depth if depth > 0 else np.random.randint(1, 4)
    for _ in range(depth):
      op = np.random.choice(augmentations.augmentations)
      image_aug = apply_op(image_aug, op, severity)
    # Preprocessing commutes since all coefficients are convex
    mix += ws[i] * normalize(image_aug)

  mixed = (1 - m) * normalize(image) + m * mix
  return mixed

<commit_msg>Don't overwrite depth in numpy implementation<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Reference implementation of AugMix's data augmentation method in numpy."""
import augmentations
import numpy as np
from PIL import Image

# CIFAR-10 constants
MEAN = [0.4914, 0.4822, 0.4465]
STD = [0.2023, 0.1994, 0.2010]

def normalize(image):
  """Normalize input image channel-wise to zero mean and unit variance."""
  image = image.transpose(2, 0, 1)  # Switch to channel-first
  mean, std = np.array(MEAN), np.array(STD)
  image = (image - mean[:, None, None]) / std[:, None, None]
  return image.transpose(1, 2, 0)

def apply_op(image, op, severity):
  image = np.clip(image * 255., 0, 255).astype(np.uint8)
  pil_img = Image.fromarray(image)  # Convert to PIL.Image
  pil_img = op(pil_img, severity)
  return np.asarray(pil_img) / 255.

def augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.):
  """Perform AugMix augmentations and compute mixture.

  Args:
    image: Raw input image as float32 np.ndarray of shape (h, w, c)
    severity: Severity of underlying augmentation operators (between 1 to 10).
    width: Width of augmentation chain
    depth: Depth of augmentation chain. -1 enables stochastic depth uniformly
      from [1, 3]
    alpha: Probability coefficient for Beta and Dirichlet distributions.

  Returns:
    mixed: Augmented and mixed image.
  """
  ws = np.float32(
      np.random.dirichlet([alpha] * width))
  m = np.float32(np.random.beta(alpha, alpha))

  mix = np.zeros_like(image)
  for i in range(width):
    image_aug = image.copy()
    d = depth if depth > 0 else np.random.randint(1, 4)
    for _ in range(d):
      op = np.random.choice(augmentations.augmentations)
      image_aug = apply_op(image_aug, op, severity)
    # Preprocessing commutes since all coefficients are convex
    mix += ws[i] * normalize(image_aug)

  mixed = (1 - m) * normalize(image) + m * mix
  return mixed

<|endoftext|>


<commit_before>from rest_framework import serializers

from matches.models import Match, GameState, ResourceState, MilitaryState, TechnologyState, TerritoryState

class ResourceStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = ResourceState
        fields = (
            'resource',
            'quantity',
        )

class MilitaryStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = MilitaryState
        fields = (
            'unit',
            'quantity',
        )

class TechnologyStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = TechnologyState
        fields = (
            'technology',
            'quantity',
        )

class TerritoryStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = TerritoryState
        fields = (
            'territory',
            'player',
            'status',
        )

class GameStateSerializer(serializers.ModelSerializer):
    resources = ResourceStateSerializer(many=True, read_only=True)
    military = MilitaryStateSerializer(many=True, read_only=True)
    technology = TechnologyStateSerializer(many=True, read_only=True)
    territory = TerritoryStateSerializer(many=True, read_only=True)

    class Meta:
        model = GameState
        fields = (
            'player',
            'resources',
            'military',
            'technology',
            'territory',
        )

class MatchSerializer(serializers.ModelSerializer):
    states = GameStateSerializer(many=True, read_only=True)

    class Meta:
        model = Match
        fields = (
            'arena',
            'player_1',
            'player_2',
            'timestamp',
            'uuid',
            'completed',
            'victor',
            'get_turn_count',
        )<commit_msg>updated match serializers<commit_after>from rest_framework import serializers

from matches.models import Match, GameState, ResourceState, MilitaryState, TechnologyState, TerritoryState

class ResourceStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = ResourceState
        fields = (
            'resource',
            'quantity',
        )

class MilitaryStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = MilitaryState
        fields = (
            'unit',
            'quantity',
        )

class TechnologyStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = TechnologyState
        fields = (
            'technology',
            'quantity',
        )

class TerritoryStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = TerritoryState
        fields = (
            'territory',
            'player',
            'status',
        )

class GameStateSerializer(serializers.ModelSerializer):
    resources = ResourceStateSerializer(many=True)
    military = MilitaryStateSerializer(many=True)
    technology = TechnologyStateSerializer(many=True)
    territory = TerritoryStateSerializer(many=True)

    class Meta:
        model = GameState
        fields = (
            'player',
            'resources',
            'military',
            'technology',
            'territory',
        )

class MatchSerializer(serializers.ModelSerializer):
    uuid = serializers.SerializerMethodField()

    class Meta:
        model = Match
        fields = (
            'arena',
            'player_1',
            'player_2',
            'timestamp',
            'uuid',
            'completed',
            'victor',
        )

    def get_uuid(self, obj):
        return obj.uuid
<|endoftext|>